diff --git a/src/runner/authorization-server.ts b/src/runner/authorization-server.ts index 15d0ad58..6e9a4979 100644 --- a/src/runner/authorization-server.ts +++ b/src/runner/authorization-server.ts @@ -1,4 +1,5 @@ import { promises as fs } from 'fs'; +import { formatTally, tallyChecks } from './summary'; import path from 'path'; import { ConformanceCheck, SpecVersion } from '../types'; import { @@ -70,12 +71,8 @@ export function printAuthorizationServerResults( denominator: number; warnings: number; } { - const denominator = checks.filter( - (c) => c.status === 'SUCCESS' || c.status === 'FAILURE' - ).length; - const passed = checks.filter((c) => c.status === 'SUCCESS').length; - const failed = checks.filter((c) => c.status === 'FAILURE').length; - const warnings = checks.filter((c) => c.status === 'WARNING').length; + const tally = tallyChecks(checks); + const { passed, failed, warnings, denominator } = tally; if (verbose) { console.log(JSON.stringify(checks, null, 2)); @@ -84,9 +81,7 @@ export function printAuthorizationServerResults( } console.log(`\nTest Results:`); - console.log( - `Passed: ${passed}/${denominator}, ${failed} failed, ${warnings} warnings` - ); + console.log(formatTally(tally)); if (failed > 0) { console.log('\n=== Failed Checks ==='); diff --git a/src/runner/client.ts b/src/runner/client.ts index af10657f..161441e4 100644 --- a/src/runner/client.ts +++ b/src/runner/client.ts @@ -1,4 +1,5 @@ import { spawn } from 'child_process'; +import { formatTally, tallyChecks } from './summary'; import { promises as fs } from 'fs'; import path from 'path'; import { @@ -264,12 +265,8 @@ export function printClientResults( warnings: number; overallFailure: boolean; } { - const denominator = checks.filter( - (c) => c.status === 'SUCCESS' || c.status === 'FAILURE' - ).length; - const passed = checks.filter((c) => c.status === 'SUCCESS').length; - const failed = checks.filter((c) => c.status === 'FAILURE').length; - const warnings = checks.filter((c) => c.status === 'WARNING').length; + const tally = tallyChecks(checks); + const { passed, failed, warnings, denominator } = tally; // Determine if there's an overall failure (failures, warnings, client timeout, or exit failure) const clientTimedOut = clientOutput?.timedOut ?? false; @@ -292,9 +289,7 @@ export function printClientResults( // Test results summary goes to stderr console.error(`\nTest Results:`); - console.error( - `Passed: ${passed}/${denominator}, ${failed} failed, ${warnings} warnings` - ); + console.error(formatTally(tally)); if (clientTimedOut) { console.error(`\n⚠️ CLIENT TIMED OUT - Test incomplete`); diff --git a/src/runner/server.ts b/src/runner/server.ts index 21e80ec4..eff46219 100644 --- a/src/runner/server.ts +++ b/src/runner/server.ts @@ -1,4 +1,5 @@ import { promises as fs } from 'fs'; +import { formatTally, tallyChecks } from './summary'; import path from 'path'; import { ConformanceCheck, @@ -178,12 +179,8 @@ export function printServerResults( denominator: number; warnings: number; } { - const denominator = checks.filter( - (c) => c.status === 'SUCCESS' || c.status === 'FAILURE' - ).length; - const passed = checks.filter((c) => c.status === 'SUCCESS').length; - const failed = checks.filter((c) => c.status === 'FAILURE').length; - const warnings = checks.filter((c) => c.status === 'WARNING').length; + const tally = tallyChecks(checks); + const { passed, failed, warnings, denominator } = tally; if (verbose) { console.log(JSON.stringify(checks, null, 2)); @@ -192,9 +189,7 @@ export function printServerResults( } console.log(`\nTest Results:`); - console.log( - `Passed: ${passed}/${denominator}, ${failed} failed, ${warnings} warnings` - ); + console.log(formatTally(tally)); if (failed > 0) { console.log('\n=== Failed Checks ==='); diff --git a/src/runner/summary.test.ts b/src/runner/summary.test.ts new file mode 100644 index 00000000..c7b58c9f --- /dev/null +++ b/src/runner/summary.test.ts @@ -0,0 +1,97 @@ +import { describe, test, expect } from 'vitest'; +import type { ConformanceCheck } from '../types'; +import { untestableCheck, notTestable } from '../scenarios/untestable'; +import { formatTally, isUnverified, tallyChecks } from './summary'; + +const check = ( + id: string, + status: ConformanceCheck['status'], + extra: Partial = {} +): ConformanceCheck => ({ + id, + name: id, + description: id, + status, + timestamp: '2026-09-24T00:00:00.000Z', + specReferences: [], + ...extra +}); + +const ref = [{ specVersion: 'draft', section: 'x' }] as never; + +describe('tallyChecks', () => { + test('counts the same four numbers the runners counted before', () => { + const tally = tallyChecks([ + check('a', 'SUCCESS'), + check('b', 'SUCCESS'), + check('c', 'FAILURE'), + check('d', 'WARNING'), + check('e', 'SKIPPED') + ]); + expect(tally).toMatchObject({ + passed: 2, + failed: 1, + warnings: 1, + denominator: 3 + }); + }); + + test('unverified is a subset of failed and warnings, never an addition', () => { + const tally = tallyChecks([ + check('violated', 'FAILURE', { errorMessage: 'cursor was 42' }), + untestableCheck('missing', 'missing', 'd', 'no control', ref), + untestableCheck('soft', 'soft', 'd', 'no control', ref, 'WARNING') + ]); + expect(tally.failed).toBe(2); + expect(tally.warnings).toBe(1); + expect(tally.unverified).toBe(2); + // The verdict a runner keys its exit code off is unchanged by the split. + expect(tally.failed + tally.warnings).toBeGreaterThan(0); + }); + + test('a scenario that formats its own message still counts as unverified', () => { + const own = check('own', 'FAILURE', { + errorMessage: notTestable('the fixture exposes no control'), + details: {} + }); + expect(isUnverified(own)).toBe(true); + expect(tallyChecks([own]).unverified).toBe(1); + }); + + test('a skipped check is neither failed nor unverified', () => { + const skipped = check('skip', 'SKIPPED', { + errorMessage: notTestable('does not apply') + }); + expect(isUnverified(skipped)).toBe(false); + expect(tallyChecks([skipped])).toMatchObject({ + failed: 0, + unverified: 0, + denominator: 0 + }); + }); +}); + +describe('formatTally', () => { + test('prints what it always printed when nothing is unverified', () => { + const line = formatTally({ + passed: 9, + failed: 1, + warnings: 0, + denominator: 10, + unverified: 0 + }); + expect(line).toBe('Passed: 9/10, 1 failed, 0 warnings'); + }); + + test('says how much of a red run was never checked', () => { + const line = formatTally({ + passed: 2, + failed: 27, + warnings: 0, + denominator: 29, + unverified: 27 + }); + expect(line).toContain('27 failed'); + expect(line).toContain('27 of those unverified, not violated'); + }); +}); diff --git a/src/runner/summary.ts b/src/runner/summary.ts new file mode 100644 index 00000000..57591810 --- /dev/null +++ b/src/runner/summary.ts @@ -0,0 +1,75 @@ +/** + * One tally for every runner's results line. + * + * The `server`, `client` and `authorization-server` runners each counted the + * same four numbers from the same filters, so a fifth number had to be added in + * three places or in none. + * + * That fifth number is what a reader of a red run most wants: how much of it was + * a requirement being violated, and how much was a requirement that could not be + * checked at all. Those are different facts with the same status, because a + * missing prerequisite reports as a failure on purpose (issue #248) — SKIPPED is + * excluded from counts, exit codes and the expected-failures baseline, so a + * skipped row is one nobody can burn down. Splitting them in the output costs + * nothing and changes no verdict: `unverified` counts rows already inside + * `failed` and `warnings`, so exit codes stay exactly as they were. + */ + +import type { ConformanceCheck } from '../types'; +import { notTestable } from '../scenarios/untestable'; + +/** The prefix `notTestable()` writes, for checks that carry no details flag. */ +const NOT_TESTABLE_PREFIX = notTestable('').trim(); + +export interface CheckTally { + /** Checks that passed. */ + passed: number; + /** Checks with status FAILURE, including ones that could not be exercised. */ + failed: number; + /** Checks with status WARNING, including ones that could not be exercised. */ + warnings: number; + /** SUCCESS + FAILURE, the population the pass ratio is quoted against. */ + denominator: number; + /** + * Of `failed` + `warnings`, how many report a missing prerequisite rather + * than a violated requirement. A subset, never an addition. + */ + unverified: number; +} + +/** + * Whether a check reports a prerequisite it could not satisfy. + * + * Either signal counts: `untestableCheck()` sets `details.untestable`, and a + * scenario that formats its own message with `notTestable()` carries only the + * prefix. Reading both means a scenario cannot fall out of the tally by + * building its check the other way. + */ +export function isUnverified(check: ConformanceCheck): boolean { + if (check.status !== 'FAILURE' && check.status !== 'WARNING') return false; + if (check.details?.untestable === true) return true; + return (check.errorMessage ?? '').startsWith(NOT_TESTABLE_PREFIX); +} + +export function tallyChecks(checks: ConformanceCheck[]): CheckTally { + return { + passed: checks.filter((c) => c.status === 'SUCCESS').length, + failed: checks.filter((c) => c.status === 'FAILURE').length, + warnings: checks.filter((c) => c.status === 'WARNING').length, + denominator: checks.filter( + (c) => c.status === 'SUCCESS' || c.status === 'FAILURE' + ).length, + unverified: checks.filter(isUnverified).length + }; +} + +/** + * The results line. The unverified clause is appended only when there is one, + * so a run with nothing unverified prints exactly what it always printed. + */ +export function formatTally(tally: CheckTally): string { + const base = `Passed: ${tally.passed}/${tally.denominator}, ${tally.failed} failed, ${tally.warnings} warnings`; + return tally.unverified > 0 + ? `${base} (${tally.unverified} of those unverified, not violated)` + : base; +} diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index ff377f54..4a0b1f80 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -60,6 +60,11 @@ import { ResourcesNotFoundErrorScenario } from './server/resources'; +import { EventsDiscoveryScenario } from './server/events/discovery'; +import { EventsPollScenario } from './server/events/poll'; +import { EventsPushScenario } from './server/events/push'; +import { EventsWebhookScenario } from './server/events/webhook'; +import { EventsWebhookDeliveryScenario } from './server/events/webhook-delivery'; import { SkillsDirectoryReadScenario } from './server/skills/directory'; import { SkillsEnumerationScenario } from './server/skills/enumeration'; import { SkillsManifestScenario } from './server/skills/manifest'; @@ -166,7 +171,15 @@ const pendingClientScenariosList: ClientScenario[] = [ // `npm start -- server --scenario sep-2640-skills-* --url `. new SkillsDirectoryReadScenario(), new SkillsEnumerationScenario(), - new SkillsManifestScenario() + new SkillsManifestScenario(), + + // MCP Events. Pending because the everything-server does not implement the + // events capability; targeted runs point at an events-capable fixture via + // `npm start -- server --scenario events-* --url `. The suite + // scores against SEP-3415 (modelcontextprotocol/modelcontextprotocol#3415); + // see the header of src/seps/sep-3415.yaml. + new EventsDiscoveryScenario(), + new EventsPollScenario() ]; // All client scenarios @@ -224,6 +237,15 @@ const allClientScenariosList: ClientScenario[] = [ new SkillsEnumerationScenario(), new SkillsManifestScenario(), + // MCP Events. Fixture-dependent (needs a server declaring the extension under + // `capabilities.extensions`); + // each scenario SKIPs cleanly when the capability is not declared. + new EventsDiscoveryScenario(), + new EventsPollScenario(), + new EventsPushScenario(), + new EventsWebhookScenario(), + new EventsWebhookDeliveryScenario(), + // Prompts scenarios new PromptsListScenario(), new PromptsGetSimpleScenario(), diff --git a/src/scenarios/server/events/discovery.ts b/src/scenarios/server/events/discovery.ts new file mode 100644 index 00000000..62a6a178 --- /dev/null +++ b/src/scenarios/server/events/discovery.ts @@ -0,0 +1,839 @@ +/** + * MCP Events — capability declaration, `events/list`, and the error-code + * contract. + * + * One scenario, many checks (per AGENTS.md "fewer scenarios, more checks"). + * Each check's verbatim spec excerpt lives next to its check ID in + * src/seps/sep-3415.yaml, keeping the yaml and this scenario in lock-step. + * + * This is the gate scenario for the suite: the delivery-mode scenarios all + * start from a descriptor found here, so a server that fails `events/list` + * fails everything downstream for a reason this scenario names. + * + * Discovery is dynamic and brand-neutral. The scenario enumerates whatever the + * server serves and validates the descriptors it finds, hardcoding no event + * name. An empty catalog is permitted by the document, so descriptor-level + * checks report the unmet prerequisite via `untestableCheck` rather than + * passing vacuously. + * + * The capability gate is two-sided rather than a plain SKIP. An optional + * capability a server never declared is not a defect, so a server that also + * does not implement `events/list` skips the whole scenario. A server that + * answers `events/list` while declaring nothing is a different thing: it has + * an events surface that a client following the spec would never call, and + * SKIP would report that as a clean run. So the scenario asks before it + * skips. + */ + +import { ClientScenario, ConformanceCheck } from '../../../types'; +import type { RunContext } from '../../../connection'; +import { untestableCheck } from '../../untestable'; +import { openEventStream } from './stream'; +import { + EVENTS_CONTROL_CATALOG, + EVENTS_LIST_CHANGED_FILTER_KEY, + EVENTS_LIST_CHANGED_NOTIFICATION, + SUBSCRIPTIONS_ACKNOWLEDGED_NOTIFICATION, + SUBSCRIPTIONS_LISTEN_METHOD, + askControl, + hasControl, + EVENTS_EXTENSION_ID, + EVENTS_LIST_METHOD, + EVENTS_POLL_METHOD, + EVENTS_SPEC_REF, + EVENTS_NOT_FOUND, + EVENTS_UNSUPPORTED, + DELIVERY_MODES, + JSONRPC_METHOD_NOT_FOUND, + type EventDescriptor, + declaredEventsCapability, + eventsCapability, + eventsCheck, + eventsListPage, + eventsListAll, + eventsPoll, + deliveryModes, + descriptorName, + descriptorLabel, + describeValue, + isObject, + isEventsErrorCode, + resultTypeCheck, + RESULT_TYPE_ID, + unknownEventName +} from './helpers'; + +const CAPABILITY_IDS = [ + 'sep-3415-capability-events-object', + 'sep-3415-capability-list-changed-flag', + 'sep-3415-capability-empty-settings' +] as const; + +const LIST_IDS = [ + 'sep-3415-list-implemented', + 'sep-3415-list-pagination' +] as const; + +const DESCRIPTOR_IDS = [ + 'sep-3415-descriptor-name', + 'sep-3415-descriptor-delivery-subset', + 'sep-3415-descriptor-input-schema', + 'sep-3415-descriptor-payload-schema', + 'sep-3415-descriptor-meta' +] as const; + +const ERROR_IDS = [ + 'sep-3415-error-not-found', + 'sep-3415-error-server-range', + 'sep-3415-fallback-method-not-found' +] as const; + +const LISTEN_IDS = [ + 'sep-3415-capability-list-changed-gated', + 'sep-3415-list-changed-notification' +] as const; + +const GATED_DESCRIPTION = + 'A server that did not declare `listChanged: true` omits `eventsListChanged` from the acknowledged filter.'; +const LIST_CHANGED_DESCRIPTION = + 'A change to the set of event types sends `notifications/events/list_changed`, on a `subscriptions/listen` stream whose client opted in with `eventsListChanged`.'; + +/** How long to wait for the acknowledgement, and for the notification. */ +const LISTEN_ACK_MS = 3000; +const LIST_CHANGED_MS = 3000; + +const FALLBACK_DESCRIPTION = + 'A server that does not offer the extension answers any `events/*` request with `-32601 MethodNotFound` (standard JSON-RPC).'; + +const ALL_IDS = [ + ...CAPABILITY_IDS, + ...LIST_IDS, + ...DESCRIPTOR_IDS, + ...ERROR_IDS, + ...LISTEN_IDS, + RESULT_TYPE_ID +]; + +/** Every check this scenario can emit, as SKIPPED with one shared reason. */ +function skipAll(reason: string): ConformanceCheck[] { + return ALL_IDS.map((id) => + eventsCheck(id, reason, 'SKIPPED', { errorMessage: reason }) + ); +} + +export class EventsDiscoveryScenario implements ClientScenario { + name = 'events-discovery'; + readonly source = { extensionId: EVENTS_EXTENSION_ID } as const; + description = `MCP Events: capability declaration, \`events/list\` enumeration, and the error-code contract. + +**Methods**: \`events/list\` (mandatory for a server declaring the extension), \`events/poll\` (probed only for its error path) + +**Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-3415.yaml): + +- \`sep-3415-capability-events-object\` — \`events\` is declared top-level under \`capabilities\` as an object +- \`sep-3415-capability-list-changed-flag\` — \`listChanged\`, when present, is a boolean +- \`sep-3415-list-implemented\` — \`events/list\` is implemented and returns an \`events\` array +- \`sep-3415-list-pagination\` — \`nextCursor\` is honoured as a cursor on the next request +- \`sep-3415-descriptor-*\` — the descriptor fields: \`name\`, \`description\`, \`delivery\` as a non-empty subset of poll/push/webhook, \`inputSchema\`, \`payloadSchema\`, and \`_meta\` when present +- \`sep-3415-error-not-found\` — an unknown event name answers \`-32023 NotFound\` (the poll-specific restatement of the same rule is graded by \`events-poll\`) +- \`sep-3415-error-server-range\` — the code is one of the five SEP-3415 allocates (\`-32023\` to \`-32027\`), not the design sketch's \`-32011\` to \`-32015\` +- \`sep-3415-result-type-complete\` — the \`events/list\` result carries \`resultType: "complete"\` +- \`sep-3415-capability-list-changed-gated\` — a server that did not declare \`listChanged: true\` leaves \`eventsListChanged\` out of the \`subscriptions/listen\` acknowledgement +- \`sep-3415-list-changed-notification\` — a catalog change, made through the \`events_conformance_catalog\` control, sends \`notifications/events/list_changed\` on an opted-in listen stream + +**Discovery is dynamic**: a server that neither declares the extension nor implements \`events/list\` SKIPs everything. One that answers \`events/list\` without declaring the extension is graded, and fails the declaration check, because that surface is unreachable for a client that reads capabilities first. An empty catalog reports the descriptor checks as untestable rather than passing them.`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect(); + try { + const checks = await this.checks(conn); + // The skip path already carries the listen rows as SKIPPED; every other + // path grades them here, so no row vanishes depending on the server. + if ( + !checks.some((c) => (LISTEN_IDS as readonly string[]).includes(c.id)) + ) { + checks.push(...(await this.listenChecks(ctx, conn))); + } + return checks; + } finally { + await conn.close(); + } + } + + /** + * `list_changed` reaches a client only on a `subscriptions/listen` stream it + * opened with `eventsListChanged: true`, and the server's acknowledgement + * says whether it will honor that. Two rows hang off one stream: + * + * - the gate: a server that did not declare `listChanged: true` must leave + * the key out of the acknowledgement. A server that did declare it may + * still decline, so for it this row has nothing to grade; + * - the notification: a server that declared `listChanged: true` sends + * `notifications/events/list_changed` when its catalog changes. The + * harness cannot change a catalog over the protocol, so it asks the + * fixture's `events_conformance_catalog` control to add a type, waits for + * the notification, then removes the type again. + */ + private async listenChecks( + ctx: RunContext, + conn: Awaited> + ): Promise { + const caps = await eventsCapability(conn); + const declaredTrue = caps?.listChanged === true; + + const session = await openEventStream( + ctx.serverUrl, + ctx.specVersion, + { notifications: { [EVENTS_LIST_CHANGED_FILTER_KEY]: true } }, + { method: SUBSCRIPTIONS_LISTEN_METHOD, openTimeoutMs: LISTEN_ACK_MS } + ); + try { + const ack = session.contentType?.includes('text/event-stream') + ? await session.waitFor( + (n) => n.method === SUBSCRIPTIONS_ACKNOWLEDGED_NOTIFICATION, + LISTEN_ACK_MS + ) + : undefined; + if (!ack) { + const why = session.error + ? `\`${SUBSCRIPTIONS_LISTEN_METHOD}\` failed: ${session.error.code} ${session.error.message}` + : `\`${SUBSCRIPTIONS_LISTEN_METHOD}\` did not confirm with \`${SUBSCRIPTIONS_ACKNOWLEDGED_NOTIFICATION}\` within ${LISTEN_ACK_MS}ms (HTTP ${session.status}, ${session.contentType ?? 'no content type'})`; + return [ + declaredTrue + ? this.gatedNotApplicable() + : untestableCheck( + LISTEN_IDS[0], + LISTEN_IDS[0], + GATED_DESCRIPTION, + why, + [EVENTS_SPEC_REF], + 'FAILURE' + ), + declaredTrue + ? untestableCheck( + LISTEN_IDS[1], + LISTEN_IDS[1], + LIST_CHANGED_DESCRIPTION, + why, + [EVENTS_SPEC_REF], + 'FAILURE' + ) + : this.notificationNotApplicable() + ]; + } + + const honored = isObject(ack.params.notifications) + ? ack.params.notifications + : {}; + const echoed = honored[EVENTS_LIST_CHANGED_FILTER_KEY] === true; + const checks: ConformanceCheck[] = []; + + if (declaredTrue) { + checks.push(this.gatedNotApplicable()); + } else { + checks.push( + eventsCheck( + LISTEN_IDS[0], + GATED_DESCRIPTION, + echoed ? 'FAILURE' : 'SUCCESS', + echoed + ? { + errorMessage: `The server declared \`listChanged: ${String(caps?.listChanged ?? false)}\` but acknowledged \`${EVENTS_LIST_CHANGED_FILTER_KEY}\`, promising a notification it did not advertise.`, + details: { listChanged: caps?.listChanged, honored } + } + : { details: { listChanged: caps?.listChanged, honored } } + ) + ); + } + + if (!declaredTrue) { + checks.push(this.notificationNotApplicable()); + return checks; + } + if (!echoed) { + checks.push( + eventsCheck(LISTEN_IDS[1], LIST_CHANGED_DESCRIPTION, 'FAILURE', { + errorMessage: `The server declared \`listChanged: true\` but did not honor \`${EVENTS_LIST_CHANGED_FILTER_KEY}\` on \`${SUBSCRIPTIONS_LISTEN_METHOD}\`, which is the only channel the notification travels on, so no client can receive it.`, + details: { honored } + }) + ); + return checks; + } + if (!(await hasControl(conn, EVENTS_CONTROL_CATALOG))) { + checks.push( + untestableCheck( + LISTEN_IDS[1], + LISTEN_IDS[1], + LIST_CHANGED_DESCRIPTION, + `The server exposes no \`${EVENTS_CONTROL_CATALOG}\` control, and the harness has no other way to change its catalog mid-run.`, + [EVENTS_SPEC_REF], + 'FAILURE' + ) + ); + return checks; + } + + const before = session.notifications.length; + const added = await askControl(conn, EVENTS_CONTROL_CATALOG, { + op: 'add' + }); + const name = added?.match(/^ok:\s*(\S+)/)?.[1]; + if (!name) { + checks.push( + untestableCheck( + LISTEN_IDS[1], + LISTEN_IDS[1], + LIST_CHANGED_DESCRIPTION, + `\`${EVENTS_CONTROL_CATALOG}\` {op: "add"} did not add a type (answered ${JSON.stringify(added)}).`, + [EVENTS_SPEC_REF], + 'FAILURE' + ) + ); + return checks; + } + const hit = await session.waitFor( + (n) => + n.method === EVENTS_LIST_CHANGED_NOTIFICATION && + session.notifications.indexOf(n) >= before, + LIST_CHANGED_MS + ); + checks.push( + eventsCheck( + LISTEN_IDS[1], + LIST_CHANGED_DESCRIPTION, + hit ? 'SUCCESS' : 'FAILURE', + hit + ? { details: { added: name, atMs: hit.atMs } } + : { + errorMessage: `Added event type \`${name}\` through \`${EVENTS_CONTROL_CATALOG}\`, and no \`${EVENTS_LIST_CHANGED_NOTIFICATION}\` arrived on the acknowledged listen stream within ${LIST_CHANGED_MS}ms.`, + details: { added: name } + } + ) + ); + await askControl(conn, EVENTS_CONTROL_CATALOG, { op: 'remove', name }); + return checks; + } finally { + await session.cancel(); + } + } + + private gatedNotApplicable(): ConformanceCheck { + return eventsCheck(LISTEN_IDS[0], GATED_DESCRIPTION, 'SKIPPED', { + errorMessage: + 'The server declared `listChanged: true`; the rule covers servers that did not, and a declaring server may echo or decline the key.' + }); + } + + private notificationNotApplicable(): ConformanceCheck { + return eventsCheck(LISTEN_IDS[1], LIST_CHANGED_DESCRIPTION, 'SKIPPED', { + errorMessage: + 'The server did not declare `listChanged: true`, so it does not promise the notification.' + }); + } + + private async checks( + conn: Awaited> + ): Promise { + const checks: ConformanceCheck[] = []; + + // --- Capability ------------------------------------------------------ + const capDescription = + "Events is declared through Extension Negotiation: the identifier appears as a key in the `extensions` field of capabilities, mapped to the extension's settings object."; + const { declared, value } = await declaredEventsCapability(conn); + /** Set only when the server declared nothing, which is the one case the + * fallback rule applies to. */ + let undeclaredProbe: Awaited> | undefined; + + if (!declared) { + // An undeclared optional capability is normally a SKIP. It is not one + // when the server answers `events/list` anyway: that server has an + // events surface no spec-following client can discover, and SKIP would + // report it as a clean run. Distinguish the two by asking. + const probe = await eventsListPage(conn); + if ('error' in probe && probe.error.code === JSONRPC_METHOD_NOT_FOUND) { + // Not an events server, so every rule about events is inapplicable — + // except the one that says what such a server answers, which it just + // did. Grading it here is the only place the suite ever meets a server + // that does not offer the extension. + const skipped = skipAll( + 'Server does not declare the `events` capability and does not implement `events/list`; the extension is optional.' + ).filter((c) => c.id !== 'sep-3415-fallback-method-not-found'); + return [ + ...skipped, + eventsCheck( + 'sep-3415-fallback-method-not-found', + FALLBACK_DESCRIPTION, + 'SUCCESS', + { details: { code: probe.error.code } } + ) + ]; + } + undeclaredProbe = probe; + checks.push( + eventsCheck( + 'sep-3415-capability-events-object', + capDescription, + 'FAILURE', + { + errorMessage: `Server answers \`${EVENTS_LIST_METHOD}\` but declares no \`capabilities.extensions["${EVENTS_EXTENSION_ID}"]\`. A client that follows the spec reads capabilities to decide whether to call it, so this surface is unreachable.`, + details: { extensionId: EVENTS_EXTENSION_ID, declared: false } + } + ) + ); + } else if (isObject(value)) { + checks.push( + eventsCheck( + 'sep-3415-capability-events-object', + capDescription, + 'SUCCESS' + ) + ); + } else { + checks.push( + eventsCheck( + 'sep-3415-capability-events-object', + capDescription, + 'FAILURE', + { + errorMessage: `\`capabilities.extensions["${EVENTS_EXTENSION_ID}"]\` is ${describeValue(value)}, expected an object.`, + details: { declared: value } + } + ) + ); + } + + const caps = declared ? await eventsCapability(conn) : undefined; + const listChanged = caps?.listChanged; + if (listChanged === undefined) { + checks.push( + eventsCheck( + 'sep-3415-capability-list-changed-flag', + 'The `listChanged` flag advertises that the server sends `notifications/events/list_changed`.', + 'SKIPPED', + { + errorMessage: declared + ? 'Server did not declare `listChanged`; the flag is optional and its absence means the notification is not advertised.' + : 'Server declared no settings object for the flag to sit in; see sep-3415-capability-events-object.' + } + ) + ); + } else if (typeof listChanged === 'boolean') { + checks.push( + eventsCheck( + 'sep-3415-capability-list-changed-flag', + 'The `listChanged` flag advertises that the server sends `notifications/events/list_changed`.', + 'SUCCESS', + { details: { listChanged } } + ) + ); + } else { + checks.push( + eventsCheck( + 'sep-3415-capability-list-changed-flag', + 'The `listChanged` flag advertises that the server sends `notifications/events/list_changed`.', + 'FAILURE', + { + errorMessage: `\`listChanged\` in the extension's settings object is ${describeValue(listChanged)}, expected a boolean.`, + details: { listChanged } + } + ) + ); + } + + checks.push(this.emptySettingsCheck(declared, value)); + checks.push(this.fallbackCheck(undeclaredProbe)); + + // --- events/list ----------------------------------------------------- + const firstPage = await eventsListPage(conn); + if ('error' in firstPage) { + const err = firstPage.error; + const unimplemented = err.code === JSONRPC_METHOD_NOT_FOUND; + checks.push( + eventsCheck( + 'sep-3415-list-implemented', + '`events/list` returns `{events: [...], nextCursor}`, where each entry describes one event type.', + 'FAILURE', + { + errorMessage: unimplemented + ? `Server declares \`capabilities.extensions["${EVENTS_EXTENSION_ID}"]\` but \`${EVENTS_LIST_METHOD}\` is not implemented (-32601).` + : `\`${EVENTS_LIST_METHOD}\` failed: ${err.code} ${err.message}`, + details: { code: err.code, message: err.message, data: err.data } + } + ) + ); + // Nothing downstream can be graded without a catalog. + const reason = `\`${EVENTS_LIST_METHOD}\` did not return a result (${err.code} ${err.message}).`; + for (const id of [...LIST_IDS.slice(1), ...DESCRIPTOR_IDS]) { + checks.push( + untestableCheck(id, id, id, reason, [EVENTS_SPEC_REF], 'FAILURE') + ); + } + checks.push(resultTypeCheck([])); + checks.push(...(await this.errorChecks(conn, []))); + return checks; + } + + const eventsField = firstPage.result.events; + if (Array.isArray(eventsField)) { + checks.push( + eventsCheck( + 'sep-3415-list-implemented', + '`events/list` returns `{events: [...], nextCursor}`, where each entry describes one event type.', + 'SUCCESS', + { details: { firstPageCount: firstPage.descriptors.length } } + ) + ); + } else { + checks.push( + eventsCheck( + 'sep-3415-list-implemented', + '`events/list` returns `{events: [...], nextCursor}`, where each entry describes one event type.', + 'FAILURE', + { + errorMessage: `\`${EVENTS_LIST_METHOD}\` result \`events\` is ${describeValue(eventsField)}, expected an array.`, + details: { events: eventsField } + } + ) + ); + } + + checks.push( + resultTypeCheck([ + { method: EVENTS_LIST_METHOD, result: firstPage.result } + ]) + ); + checks.push(await this.paginationCheck(conn, firstPage.result.nextCursor)); + + const all = await eventsListAll(conn); + const descriptors = + 'error' in all ? firstPage.descriptors : all.descriptors; + checks.push(...this.descriptorChecks(descriptors)); + checks.push(...(await this.errorChecks(conn, descriptors))); + + return checks; + } + + /** + * `nextCursor` is only gradeable when the server actually paginates. A + * single-page catalog leaves nothing to follow, which is a missing + * prerequisite rather than a pass: the document defers the semantics to the + * base protocol, so the only thing this check can establish is that + * `events/list` participates in the scheme at all. + */ + /** + * What a server that does not serve events answers to an `events/*` request. + * + * Only reachable on a server that declares nothing, since one that offers the + * extension has no business answering `-32601`. The skip path above grades the + * clean case; this one grades a server that answered something else, which + * leaves a client unable to tell "no such extension" from a real failure. + */ + private fallbackCheck( + probe: Awaited> | undefined + ): ConformanceCheck { + const id = 'sep-3415-fallback-method-not-found'; + if (!probe) { + return eventsCheck(id, FALLBACK_DESCRIPTION, 'SKIPPED', { + errorMessage: + 'The server declares the extension, so it is not a server that does not offer it and the fallback does not apply. Only a server declaring nothing can answer this rule.' + }); + } + if (!('error' in probe)) { + return eventsCheck(id, FALLBACK_DESCRIPTION, 'SKIPPED', { + errorMessage: `\`${EVENTS_LIST_METHOD}\` returned a catalog, so this server does offer the extension and the fallback does not apply to it.` + }); + } + return eventsCheck(id, FALLBACK_DESCRIPTION, 'FAILURE', { + errorMessage: `A server declaring no events capability answered \`${EVENTS_LIST_METHOD}\` with ${probe.error.code} ${probe.error.message}, where a server that does not offer the extension answers ${JSONRPC_METHOD_NOT_FOUND} MethodNotFound.`, + details: { code: probe.error.code } + }); + } + + /** + * `{}` as the settings object: support declared, no list-change notifications. + * + * Inapplicable rather than failed when the server declared settings of its + * own, because the rule describes what an empty object means and says nothing + * about a populated one. + */ + private emptySettingsCheck( + declared: boolean, + value: unknown + ): ConformanceCheck { + const id = 'sep-3415-capability-empty-settings'; + const description = + 'An empty settings object declares event support with no list-change notifications.'; + if (!declared || !isObject(value)) { + return eventsCheck(id, description, 'SKIPPED', { + errorMessage: + 'No settings object was declared, so there is no empty-object case to grade; see sep-3415-capability-events-object.' + }); + } + const keys = Object.keys(value); + if (keys.length > 0) { + return eventsCheck(id, description, 'SKIPPED', { + errorMessage: `The server declared settings (${keys.join(', ')}), so the empty-object case does not apply to it.` + }); + } + return eventsCheck(id, description, 'SUCCESS', { + details: { + note: 'Empty settings, and the catalog is still served; sep-3415-list-implemented grades that half.' + } + }); + } + + private async paginationCheck( + conn: Awaited>, + nextCursor: unknown + ): Promise { + const id = 'sep-3415-list-pagination'; + const description = + '`nextCursor` is present when more pages are available; same semantics as tools/list.'; + + if (nextCursor === undefined || nextCursor === null) { + return untestableCheck( + id, + id, + description, + `Server returned a single page from \`${EVENTS_LIST_METHOD}\` with no \`nextCursor\`, so cursor round-tripping could not be exercised.`, + [EVENTS_SPEC_REF], + 'WARNING' + ); + } + + if (typeof nextCursor !== 'string' || nextCursor.length === 0) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `\`nextCursor\` is ${describeValue(nextCursor)}, expected a non-empty string.`, + details: { nextCursor } + }); + } + + const second = await eventsListPage(conn, nextCursor); + if ('error' in second) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `Server returned \`nextCursor\` but rejected it on the next \`${EVENTS_LIST_METHOD}\`: ${second.error.code} ${second.error.message}`, + details: { nextCursor, code: second.error.code } + }); + } + + if (second.result.nextCursor === nextCursor) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: + 'Server echoed the same `nextCursor` on the following page, which never terminates.', + details: { nextCursor } + }); + } + + return eventsCheck(id, description, 'SUCCESS', { + details: { secondPageCount: second.descriptors.length } + }); + } + + /** + * Grade every descriptor and report one check per field, naming the first + * offender. One check per field rather than per descriptor keeps the check + * IDs stable across servers with different catalog sizes. + */ + private descriptorChecks(descriptors: EventDescriptor[]): ConformanceCheck[] { + if (descriptors.length === 0) { + const reason = `Server's \`${EVENTS_LIST_METHOD}\` returned an empty catalog, so no descriptor could be validated.`; + return DESCRIPTOR_IDS.map((id) => + untestableCheck(id, id, id, reason, [EVENTS_SPEC_REF], 'FAILURE') + ); + } + + const out: ConformanceCheck[] = []; + + const field = ( + id: string, + description: string, + severity: 'FAILURE' | 'WARNING', + predicate: (d: EventDescriptor) => string | undefined + ) => { + for (const [i, d] of descriptors.entries()) { + const problem = predicate(d); + if (problem) { + out.push( + eventsCheck(id, description, severity, { + errorMessage: `${descriptorLabel(d, i)}: ${problem}`, + details: { descriptor: d } + }) + ); + return; + } + } + out.push( + eventsCheck(id, description, 'SUCCESS', { + details: { descriptorsChecked: descriptors.length } + }) + ); + }; + + field( + 'sep-3415-descriptor-name', + 'Each event descriptor carries a `name` identifying the event type.', + 'FAILURE', + (d) => + descriptorName(d) === undefined + ? `\`name\` is ${describeValue(d.name)}, expected a non-empty string.` + : undefined + ); + + field( + 'sep-3415-descriptor-delivery-subset', + '`delivery` lists the delivery modes this event type supports — any non-empty subset of `poll`, `push`, `webhook`.', + 'FAILURE', + (d) => { + if (!Array.isArray(d.delivery)) { + return `\`delivery\` is ${describeValue(d.delivery)}, expected an array.`; + } + const modes = deliveryModes(d); + if (modes.length === 0) + return '`delivery` is empty; the subset must be non-empty.'; + const unknown = modes.filter( + (m) => !(DELIVERY_MODES as readonly string[]).includes(m) + ); + if (unknown.length > 0) { + return `\`delivery\` contains ${unknown.map((m) => `\`${m}\``).join(', ')}, outside the poll/push/webhook set.`; + } + if (new Set(modes).size !== modes.length) { + return '`delivery` repeats a mode; it is a subset, not a list.'; + } + return undefined; + } + ); + + field( + 'sep-3415-descriptor-input-schema', + '`inputSchema` is a JSON Schema describing valid subscription arguments.', + 'FAILURE', + (d) => + isObject(d.inputSchema) + ? undefined + : `\`inputSchema\` is ${describeValue(d.inputSchema)}, expected a JSON Schema object.` + ); + + field( + 'sep-3415-descriptor-payload-schema', + '`payloadSchema` describes the shape of `data` in delivered events.', + 'FAILURE', + (d) => + isObject(d.payloadSchema) + ? undefined + : `\`payloadSchema\` is ${describeValue(d.payloadSchema)}, expected a JSON Schema object.` + ); + + field( + 'sep-3415-descriptor-meta', + '`_meta` on an event descriptor is optional; same semantics as on Tool/Resource/Prompt.', + 'WARNING', + (d) => + d._meta === undefined || isObject(d._meta) + ? undefined + : `\`_meta\` is ${describeValue(d._meta)}, expected an object when present.` + ); + + return out; + } + + /** + * Probe the unknown-name error path through `events/poll`. + * + * Poll is the cheapest probe: it holds no server-side state, so a rejected + * call leaves nothing behind. The document states the same obligation twice, + * once as the general `-32023 NotFound` code and once as the poll-specific + * consequence of an event type having been removed, so both rows are graded + * from this one exchange rather than by poking the server twice. + */ + private async errorChecks( + conn: Awaited>, + descriptors: EventDescriptor[] + ): Promise { + const notFoundDesc = + '`-32023 NotFound` — a referenced entity does not exist, such as an unknown event name.'; + const rangeDesc = + 'The core specification partitions the JSON-RPC implementation-defined range: `-32000` to `-32019` is left to implementations, and `-32020` to `-32099` is allocated sequentially by the specification. This SEP requests the next five codes in the allocated range.'; + + // A server offering no poll-capable event type may legitimately not route + // events/poll at all, which would make -32601 the honest answer and this + // probe meaningless. + const pollable = descriptors.some((d) => deliveryModes(d).includes('poll')); + if (descriptors.length > 0 && !pollable) { + const reason = `No event type advertises \`poll\` delivery, so \`${EVENTS_POLL_METHOD}\` could not be used to probe the unknown-name error path.`; + return [ + untestableCheck( + 'sep-3415-error-not-found', + 'sep-3415-error-not-found', + notFoundDesc, + reason, + [EVENTS_SPEC_REF], + 'FAILURE' + ), + untestableCheck( + 'sep-3415-error-server-range', + 'sep-3415-error-server-range', + rangeDesc, + reason, + [EVENTS_SPEC_REF], + 'FAILURE' + ) + ]; + } + + const name = unknownEventName(); + const probe = await eventsPoll(conn, { name, arguments: {}, cursor: null }); + + if (!('error' in probe)) { + const errorMessage = `\`${EVENTS_POLL_METHOD}\` for unknown event name \`${name}\` returned a result instead of an error.`; + return [ + eventsCheck('sep-3415-error-not-found', notFoundDesc, 'FAILURE', { + errorMessage, + details: { result: probe.result } + }), + eventsCheck('sep-3415-error-server-range', rangeDesc, 'FAILURE', { + errorMessage + }) + ]; + } + + const { code, message, data } = probe.error; + const isNotFound = code === EVENTS_NOT_FOUND; + const details = { code, message, data, probedName: name }; + + const out: ConformanceCheck[] = []; + + out.push( + isNotFound + ? eventsCheck('sep-3415-error-not-found', notFoundDesc, 'SUCCESS', { + details + }) + : eventsCheck('sep-3415-error-not-found', notFoundDesc, 'FAILURE', { + errorMessage: `Unknown event name answered ${code}, expected ${EVENTS_NOT_FOUND} NotFound.${ + code === EVENTS_UNSUPPORTED + ? ' `-32026 Unsupported` is for a well-formed request naming an option the server does not offer, not for a name it does not have.' + : '' + }`, + details + }) + ); + + out.push( + isEventsErrorCode(code) + ? eventsCheck('sep-3415-error-server-range', rangeDesc, 'SUCCESS', { + details + }) + : eventsCheck('sep-3415-error-server-range', rangeDesc, 'FAILURE', { + errorMessage: + code <= -32000 && code >= -32019 + ? `Error code ${code} sits in \`-32000\` to \`-32019\`, which the specification leaves to implementations. SEP-3415 allocates \`-32023\` to \`-32027\`; \`-32011\` to \`-32015\` are the design sketch's numbers.` + : `Error code ${code} is not one of the five codes SEP-3415 allocates (\`-32023\` to \`-32027\`).`, + details + }) + ); + + return out; + } +} + +/** Exported for the negative tests, which assert the full emitted set. */ +export const EVENTS_DISCOVERY_CHECK_IDS = ALL_IDS; diff --git a/src/scenarios/server/events/helpers.ts b/src/scenarios/server/events/helpers.ts new file mode 100644 index 00000000..10e9cc5d --- /dev/null +++ b/src/scenarios/server/events/helpers.ts @@ -0,0 +1,668 @@ +/** + * Shared helpers for the MCP Events server-conformance scenarios under this + * directory. + * + * Extracted against the merged design sketch on `main` of + * modelcontextprotocol/experimental-ext-triggers-events (merged 2026-09-08). + * Each check's verbatim excerpt lives next to its check ID in + * src/seps/sep-3415.yaml. The rows still quote the sketch while they are + * re-extracted against SEP-3415; see that file's header. + * + * One thing about Events differs from every other extension suite here and is + * worth knowing before reading the scenarios: + * + * 1. No delivery mode is mandatory. A descriptor's `delivery` array is any + * non-empty subset of poll/push/webhook, so a scenario for one mode has to + * discover whether any event type offers it before it can probe anything. + * Nothing is hardcoded to a fixture's event names. + */ + +import type { + CheckStatus, + ConformanceCheck, + SpecReference +} from '../../../types'; +import type { Connection } from '../../../connection'; +import { JsonRpcError } from '../../../connection'; +import { untestableCheck } from '../../untestable'; + +/** + * The Events extension identifier, per SEP-2133 Extension Negotiation. + * + * It does double duty: `ScenarioSource` carries it as `{ extensionId }`, which + * keeps these scenarios out of `--spec-version` selection (see + * `matchesSpecVersion` in src/scenarios/index.ts), and it is the key the + * capability itself lives under in `capabilities.extensions`. + * + * Those were two different things until 2026-09-22. The design sketch put the + * capability at the top level as `capabilities.events`, and this file said in + * as many words not to read the capability at this key. Upstream PR 7 moved the + * sketch into the extensions map after mcpkit followed the document and the + * reference server did not, so the two uses collapsed into one. + */ +export const EVENTS_EXTENSION_ID = 'io.modelcontextprotocol/events'; + +export const EVENTS_LIST_METHOD = 'events/list'; +export const EVENTS_POLL_METHOD = 'events/poll'; +export const EVENTS_STREAM_METHOD = 'events/stream'; +export const EVENTS_SUBSCRIBE_METHOD = 'events/subscribe'; +export const EVENTS_UNSUBSCRIBE_METHOD = 'events/unsubscribe'; + +export const EVENTS_LIST_CHANGED_NOTIFICATION = + 'notifications/events/list_changed'; +export const EVENTS_EVENT_NOTIFICATION = 'notifications/events/event'; +export const EVENTS_ACTIVE_NOTIFICATION = 'notifications/events/active'; +export const EVENTS_HEARTBEAT_NOTIFICATION = 'notifications/events/heartbeat'; +export const EVENTS_ERROR_NOTIFICATION = 'notifications/events/error'; +export const EVENTS_TERMINATED_NOTIFICATION = 'notifications/events/terminated'; + +/** + * The `_meta` key carrying the parent `events/stream` request id on every + * `notifications/events/*` message, per SEP-2575's correlation convention. + */ +export const SUBSCRIPTION_ID_META = 'io.modelcontextprotocol/subscriptionId'; + +/** The three delivery modes, as they appear in a descriptor's `delivery`. */ +export const DELIVERY_MODES = ['poll', 'push', 'webhook'] as const; +export type DeliveryMode = (typeof DELIVERY_MODES)[number]; + +/** Standard JSON-RPC. */ +export const JSONRPC_METHOD_NOT_FOUND = -32601; +export const JSONRPC_INVALID_PARAMS = -32602; + +/** + * The five general-purpose codes SEP-3415 requests, named for reuse across MCP + * rather than scoped to events. Each conveys its specifics through a typed + * `data` payload rather than by minting more numbers. + * + * They are the next five in the range the core specification allocates + * sequentially (`-32020` to `-32099`), and provisional until the SEP is + * accepted. The design sketch numbered them `-32011` to `-32015`, inside + * `-32000` to `-32019`, which the specification leaves to implementations and + * never assigns, so a server still sending those is grading against the + * sketch rather than the SEP. + */ +export const EVENTS_NOT_FOUND = -32023; +export const EVENTS_FORBIDDEN = -32024; +export const EVENTS_RESOURCE_EXHAUSTED = -32025; +export const EVENTS_UNSUPPORTED = -32026; +export const EVENTS_CALLBACK_ENDPOINT_ERROR = -32027; + +/** Every code SEP-3415 defines, in its Error Handling table's order. */ +export const EVENTS_ERROR_CODES: readonly number[] = [ + EVENTS_NOT_FOUND, + EVENTS_FORBIDDEN, + EVENTS_RESOURCE_EXHAUSTED, + EVENTS_UNSUPPORTED, + EVENTS_CALLBACK_ENDPOINT_ERROR +]; + +export const EVENTS_SPEC_REF: SpecReference = { + id: 'MCP-Events', + url: 'https://github.com/modelcontextprotocol/experimental-ext-triggers-events/blob/main/docs/design-sketch-proposal.md' +}; + +/** One entry of the `events` array returned by `events/list`. */ +export interface EventDescriptor { + name?: unknown; + description?: unknown; + delivery?: unknown; + inputSchema?: unknown; + payloadSchema?: unknown; + _meta?: unknown; + [key: string]: unknown; +} + +export interface EventsListResult { + events?: unknown; + nextCursor?: unknown; + [key: string]: unknown; +} + +/** One entry of a poll response's `events` array, or a pushed notification. */ +export interface EventOccurrence { + eventId?: unknown; + name?: unknown; + timestamp?: unknown; + data?: unknown; + cursor?: unknown; + _meta?: unknown; + [key: string]: unknown; +} + +export interface EventsPollResult { + events?: unknown; + cursor?: unknown; + truncated?: unknown; + hasMore?: unknown; + nextPollMs?: unknown; + [key: string]: unknown; +} + +/** + * Build a check carrying the Events spec reference. Per AGENTS.md the same + * `id` flips `status` + `errorMessage` between SUCCESS and FAILURE rather than + * branching into distinct slugs. + */ +export function eventsCheck( + id: string, + description: string, + status: CheckStatus, + extras: Partial = {} +): ConformanceCheck { + return { + id, + name: id, + description, + status, + timestamp: new Date().toISOString(), + specReferences: [EVENTS_SPEC_REF], + ...extras + }; +} + +/** A JSON object, as opposed to an array, `null`, or a primitive. */ +export function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** + * How to name an observed value in an error message. + * + * `absent` rather than `a undefined`, because a reader chasing a failure needs + * to know the field was missing, and the JavaScript spelling of that is noise. + */ +export function describeValue(value: unknown): string { + if (value === undefined) return 'absent'; + if (value === null) return 'null'; + if (Array.isArray(value)) return 'an array'; + return `a ${typeof value}`; +} + +/** + * Whether the server declared the events capability at all, and the raw value + * it declared it with, before any shape coercion. + * + * Kept separate from `eventsCapability` so callers can tell "absent" from + * "declared with the wrong type" apart. Folding the two together would turn a + * server that declares `events: true` into a clean SKIP of the whole suite, + * which reads as a green run against a server that is plainly wrong. + */ +export async function declaredEventsCapability( + conn: Connection +): Promise<{ declared: boolean; value: unknown }> { + const discovered = await conn.discover(); + const caps = (discovered.capabilities as Record) ?? {}; + const exts = extensionsOf(caps); + if (!(EVENTS_EXTENSION_ID in exts)) + return { declared: false, value: undefined }; + return { declared: true, value: exts[EVENTS_EXTENSION_ID] }; +} + +/** + * Names of the optional diagnostic controls a fixture may expose so the harness + * can provoke conditions no protocol request can ask for. + * + * Several requirements describe what a server does when something goes wrong + * upstream, and a healthy server does none of those during a run. Without a way + * to trigger them the rows report untestable forever. A fixture that wants + * those rows graded registers these as ordinary tools; one that does not is + * unaffected and keeps reporting untestable with the prerequisite named. + * + * Each control takes `{ name: }`. mcpkit's + * examples/events/kitchen-sink registers them under --conformance-events; see + * that repo's examples/CONVENTIONS.md for the convention. + */ +export const EVENTS_CONTROL_YIELD_ERROR = 'events_conformance_yield_error'; +export const EVENTS_CONTROL_YIELD_GAP = 'events_conformance_yield_gap'; +/** + * Ends every live subscription to an event type. Terminal for that type for + * the life of the fixture process, so a scenario firing it must pick a type + * nothing else in the run depends on. + */ +export const EVENTS_CONTROL_TERMINATE = 'events_conformance_terminate'; +/** + * Registers a subscription on another principal's behalf, returning its derived + * id. A run authenticates as one principal for its lifetime, so this is the + * only way to construct the two-tenant case the key-composition rule is about. + */ +export const EVENTS_CONTROL_SUBSCRIBE_AS = 'events_conformance_subscribe_as'; +/** + * Permits callbacks under one origin past the scheme and routability guards for + * the rest of the fixture process, so the harness can be delivered to. + * + * The harness necessarily listens on loopback, and a hardened server refuses a + * loopback callback — correctly, which is what the SSRF rows grade. Those two + * facts cannot both hold in one subscription, so without this the delivery rows + * are unreachable unless EVENTS_WEBHOOK_CALLBACK_BASE points at a public tunnel. + * + * Takes `{ origin }` and names one origin, not a blanket "allow private + * networks": the fixture stays hardened for every other callback, which is what + * lets the SSRF rows be graded first and then the delivery rows after. mcpkit's + * `--conformance-events` build already allowlists one origin this way for the + * events-webhook scenario, which is spec path (b) doing the same job. + */ +export const EVENTS_CONTROL_ALLOW_CALLBACK_ORIGIN = + 'events_conformance_allow_callback_origin'; + +/** + * Reports one per-event-type subscription cap the server enforces, as JSON text + * `{"name": "", "max": }`. Read-only. + * + * `-32025` is only reachable by exceeding a limit, and nothing in the protocol + * says where a server's limits are. The concurrency probe cannot find one for + * the suite: it opens three streams on one type and needs all three to stay + * open, so a server capped there fails the MUST beside it. Knowing the capped + * type lets the quota be probed on its own. + */ +export const EVENTS_CONTROL_QUOTA = 'events_conformance_quota'; + +/** + * Send a `{type:"gap"}` envelope to one webhook subscription, `{ id }`, + * answering the cursor it carried. Per subscription rather than per event type + * because a source's gap signal reaches push streams, and a webhook subscriber + * hears of one only when the server posts to it. + */ +export const EVENTS_CONTROL_WEBHOOK_GAP = 'events_conformance_webhook_gap'; + +/** + * End one webhook subscription, `{ id }`, sending it `{type:"terminated"}`. + * Per subscription so the scenario ends only its own, where terminating an + * event type would end it for every scenario after. + */ +export const EVENTS_CONTROL_WEBHOOK_TERMINATE = + 'events_conformance_webhook_terminate'; + +/** Reports whether a given principal's subscription is still registered. */ +export const EVENTS_CONTROL_SUBSCRIPTION_EXISTS = + 'events_conformance_subscription_exists'; +/** + * Rebuilds the server and its subscription registry over the same store, as a + * process restart would. Every session ends, so a scenario firing it must + * reconnect and must fire it last. + */ +/** + * Changes the event catalog at runtime: `{op: "add"}` registers a fresh type + * and answers `ok: `, `{op: "remove", name}` drops one the control + * added. Both are changes to the set of event types, so a server that + * declared `listChanged: true` sends `notifications/events/list_changed` for + * each. + */ +export const EVENTS_CONTROL_CATALOG = 'events_conformance_catalog'; + +/** The core method a client opens to receive list-changed notifications. */ +export const SUBSCRIPTIONS_LISTEN_METHOD = 'subscriptions/listen'; + +/** First frame on a listen stream; `params.notifications` is the honored filter. */ +export const SUBSCRIPTIONS_ACKNOWLEDGED_NOTIFICATION = + 'notifications/subscriptions/acknowledged'; + +/** The filter key SEP-3415 adds to `subscriptions/listen`. */ +export const EVENTS_LIST_CHANGED_FILTER_KEY = 'eventsListChanged'; + +export const EVENTS_CONTROL_RESTART = 'events_conformance_restart'; +/** + * Answers the fixture's current restart generation. The restart control + * answers the generation it is moving to, so a scenario can tell the restart + * happened whether the server keeps sessions (the old one dies) or is + * stateless (the same connection starts reaching the new build). + */ +export const EVENTS_CONTROL_GENERATION = 'events_conformance_generation'; +/** + * Takes `{ id }`, a derived subscription id, and answers `active`, + * `suspended` or `absent`. Unlike the exists control it sees a subscription + * the server has suspended after delivery failures, which is what separates + * "paused" from "dropped". + */ +export const EVENTS_CONTROL_SUBSCRIPTION_STATE = + 'events_conformance_subscription_state'; + +/** + * Fire a control that answers with text, returning the text or undefined. + * Distinct from fireControl, which only cares that the call succeeded. + */ +export async function askControl( + conn: Connection, + tool: string, + args: Record +): Promise { + try { + const res = await conn.request<{ + content?: { type?: string; text?: string }[]; + isError?: boolean; + }>('tools/call', { name: tool, arguments: args }); + if (res.isError) return undefined; + const text = (res.content ?? []).find((c) => c?.type === 'text')?.text; + return typeof text === 'string' ? text : undefined; + } catch { + return undefined; + } +} + +/** + * Whether the server exposes a given diagnostic control. + * + * Absence is the normal case and never an error: the caller falls back to + * reporting untestable with the missing prerequisite named, per + * src/scenarios/untestable.ts. + */ +export async function hasControl( + conn: Connection, + tool: string +): Promise { + try { + const res = await conn.request<{ tools?: { name?: string }[] }>( + 'tools/list' + ); + return (res.tools ?? []).some((t) => t?.name === tool); + } catch { + return false; + } +} + +/** + * Fire a diagnostic control, returning whether it ran. A server that lists the + * tool but rejects the call is treated as not having it, so a half-implemented + * control reports untestable rather than failing the requirement it was meant + * to exercise. + */ +export async function fireControl( + conn: Connection, + tool: string, + name: string +): Promise { + try { + await conn.request('tools/call', { name: tool, arguments: { name } }); + return true; + } catch { + return false; + } +} + +/** + * The `extensions` map from a capabilities object, or an empty map. + * + * Every scenario that probes the declaration goes through this rather than + * indexing `capabilities` directly, so the one place that knows where the + * capability lives is this file. + */ +export function extensionsOf( + caps: Record +): Record { + return (caps.extensions as Record) ?? {}; +} + +/** + * The events capability object, or `undefined` when the server did not declare + * it — or declared it with something that is not an object, which callers + * treat the same way. An undeclared optional capability is a SKIP. + */ +export async function eventsCapability( + conn: Connection +): Promise | undefined> { + const { value } = await declaredEventsCapability(conn); + return isObject(value) ? value : undefined; +} + +/** A single `events/list` page, kept separate so pagination can be inspected. */ +export interface EventsListPage { + result: EventsListResult; + descriptors: EventDescriptor[]; +} + +/** + * Call `events/list` once, optionally with a cursor. Returns the `JsonRpcError` + * rather than throwing, so a scenario can grade the error instead of aborting. + */ +export async function eventsListPage( + conn: Connection, + cursor?: string +): Promise { + try { + const result = await conn.request( + EVENTS_LIST_METHOD, + cursor ? { cursor } : undefined + ); + const raw = result?.events; + return { + result: result ?? {}, + descriptors: Array.isArray(raw) ? (raw as EventDescriptor[]) : [] + }; + } catch (err) { + if (err instanceof JsonRpcError) return { error: err }; + throw err; + } +} + +/** + * Every descriptor from `events/list`, paginating until `nextCursor` clears. + * + * Bounded at `maxPages` because a server that echoes the same `nextCursor` + * forever would otherwise hang the scenario rather than fail it. Hitting the + * bound is reported through `truncatedByBound` so the caller can say so instead + * of silently grading a partial catalog. + */ +export async function eventsListAll( + conn: Connection, + maxPages = 20 +): Promise< + | { descriptors: EventDescriptor[]; pages: number; truncatedByBound: boolean } + | { error: JsonRpcError } +> { + const out: EventDescriptor[] = []; + const seen = new Set(); + let cursor: string | undefined; + let pages = 0; + + do { + const page = await eventsListPage(conn, cursor); + if ('error' in page) return page; + pages += 1; + out.push(...page.descriptors); + + const next = page.result.nextCursor; + if (typeof next !== 'string' || next.length === 0) break; + // A repeated cursor is a server bug; stop rather than loop forever. The + // pagination check grades it, this helper just refuses to hang. + if (seen.has(next)) break; + seen.add(next); + cursor = next; + } while (pages < maxPages); + + return { descriptors: out, pages, truncatedByBound: pages >= maxPages }; +} + +/** The `delivery` array of a descriptor, or `[]` when it is missing/malformed. */ +export function deliveryModes(descriptor: EventDescriptor): string[] { + const d = descriptor.delivery; + return Array.isArray(d) + ? d.filter((m): m is string => typeof m === 'string') + : []; +} + +/** The first descriptor advertising `mode`, or `undefined` when none does. */ +export function firstSupporting( + descriptors: EventDescriptor[], + mode: DeliveryMode +): EventDescriptor | undefined { + return descriptors.find((d) => deliveryModes(d).includes(mode)); +} + +/** A descriptor's `name` when it is a usable string, else `undefined`. */ +export function descriptorName( + descriptor: EventDescriptor +): string | undefined { + return typeof descriptor.name === 'string' && descriptor.name.length > 0 + ? descriptor.name + : undefined; +} + +/** + * How to refer to a descriptor in an error message without assuming it has a + * usable `name` — the checks that grade `name` itself run against descriptors + * that may not. + */ +export function descriptorLabel( + descriptor: EventDescriptor, + index: number +): string { + const name = descriptorName(descriptor); + return name ? `\`${name}\`` : `events[${index}]`; +} + +/** + * Arguments that satisfy a descriptor's `inputSchema` well enough to poll with. + * + * Optional properties are left out, so `{}` means "no filtering". Required + * properties get a value derived from the schema itself: `default`, `const`, + * the first `enum` or `examples` entry, then a type-driven value (`minimum` + * for numbers, a fixed label for strings). A required property the schema + * gives nothing to go on for returns `undefined`, and the caller reports that + * as an unmet prerequisite rather than guessing values a server would then + * reject for the wrong reason. + */ +export function minimalArguments( + descriptor: EventDescriptor +): Record | undefined { + const schema = descriptor.inputSchema; + if (!isObject(schema)) return {}; + const required = Array.isArray(schema.required) ? schema.required : []; + const properties = isObject(schema.properties) ? schema.properties : {}; + const args: Record = {}; + for (const key of required) { + if (typeof key !== 'string') return undefined; + const value = schemaValue(properties[key]); + if (value === undefined) return undefined; + args[key] = value; + } + return args; +} + +function schemaValue(prop: unknown): unknown { + if (!isObject(prop)) return undefined; + if ('default' in prop) return prop.default; + if ('const' in prop) return prop.const; + if (Array.isArray(prop.enum) && prop.enum.length > 0) return prop.enum[0]; + if (Array.isArray(prop.examples) && prop.examples.length > 0) { + return prop.examples[0]; + } + switch (prop.type) { + case 'integer': + case 'number': + return typeof prop.minimum === 'number' ? prop.minimum : 1; + case 'string': + return 'mcp-conformance'; + case 'boolean': + return false; + default: + return undefined; + } +} + +/** + * Call `events/poll`, returning the `JsonRpcError` rather than throwing so the + * caller can grade error codes. + */ +export async function eventsPoll( + conn: Connection, + params: Record +): Promise<{ result: EventsPollResult } | { error: JsonRpcError }> { + try { + const result = await conn.request( + EVENTS_POLL_METHOD, + params + ); + return { result: result ?? {} }; + } catch (err) { + if (err instanceof JsonRpcError) return { error: err }; + throw err; + } +} + +/** The `events` array of a poll result, or `[]` when missing/malformed. */ +export function occurrences(result: EventsPollResult): EventOccurrence[] { + return Array.isArray(result.events) + ? (result.events as EventOccurrence[]) + : []; +} + +/** + * Whether a value is an acceptable cursor: a string, `null`, or absent. + * + * "Absent means null" is normative in both directions, so a missing field is + * not a defect and callers must not treat it as one. + */ +export function isValidCursor(value: unknown): boolean { + return value === undefined || value === null || typeof value === 'string'; +} + +/** Whether a value parses as an ISO 8601 instant. */ +export function isIso8601(value: unknown): boolean { + if (typeof value !== 'string') return false; + const t = Date.parse(value); + if (Number.isNaN(t)) return false; + // Date.parse accepts bare dates and a few non-ISO forms; require at least a + // date and a time separated by `T`, which every example in the document has. + return /^\d{4}-\d{2}-\d{2}T/.test(value); +} + +/** Whether `code` is one of the five codes SEP-3415 allocates. */ +export function isEventsErrorCode(code: number): boolean { + return EVENTS_ERROR_CODES.includes(code); +} + +/** + * A name no conformant server should be serving, for probing the "unknown + * event name" error path. Randomised so a fixture cannot accidentally define + * it, and prefixed so a human reading server logs knows where it came from. + */ +export function unknownEventName(): string { + return `conformance.nonexistent.${Math.random().toString(36).slice(2, 10)}`; +} + +export const RESULT_TYPE_ID = 'sep-3415-result-type-complete'; +export const RESULT_TYPE_DESCRIPTION = + 'Every result this extension defines is a standard `Result` and carries `resultType: "complete"`.'; + +/** One result a scenario received, named by the method that returned it. */ +export interface ObservedResult { + method: string; + result: unknown; +} + +/** + * Grade `resultType: "complete"` across the results one scenario received. + * + * Every scenario emits this one check over whatever results it already has in + * hand, so the row is graded on `events/list`, `events/poll`, the webhook + * methods and the stream's final frame without a probe of its own. Reports the + * first offender, since one missing field is enough to fail the rule. With no + * results at all it is untestable rather than passed, per AGENTS.md #248. + */ +export function resultTypeCheck(observed: ObservedResult[]): ConformanceCheck { + if (observed.length === 0) { + return untestableCheck( + RESULT_TYPE_ID, + RESULT_TYPE_ID, + RESULT_TYPE_DESCRIPTION, + 'The scenario received no events/* result to inspect.', + [EVENTS_SPEC_REF], + 'FAILURE' + ); + } + const offender = observed.find( + (o) => !isObject(o.result) || o.result.resultType !== 'complete' + ); + if (offender) { + const value = isObject(offender.result) + ? offender.result.resultType + : undefined; + return eventsCheck(RESULT_TYPE_ID, RESULT_TYPE_DESCRIPTION, 'FAILURE', { + errorMessage: `The \`${offender.method}\` result carried \`resultType\` ${describeValue(value)}, expected "complete".`, + details: { method: offender.method, resultType: value } + }); + } + return eventsCheck(RESULT_TYPE_ID, RESULT_TYPE_DESCRIPTION, 'SUCCESS', { + details: { methods: [...new Set(observed.map((o) => o.method))] } + }); +} diff --git a/src/scenarios/server/events/negative-delivery.test.ts b/src/scenarios/server/events/negative-delivery.test.ts new file mode 100644 index 00000000..e4de9d44 --- /dev/null +++ b/src/scenarios/server/events/negative-delivery.test.ts @@ -0,0 +1,954 @@ +import { describe, test, expect, vi, beforeAll, afterAll } from 'vitest'; +import { DRAFT_PROTOCOL_VERSION } from '../../../types'; +import { + descriptor, + startEventsFixture, + type DeliveryBehaviour, + type EventsFixtureOptions +} from './negative-fixture'; + +/** + * Negative controls for `events-webhook-delivery`. + * + * This is the only scenario where the harness is the server's client *and* its + * callback endpoint, so the fixture has to POST for real. It does: a + * verification challenge, then a signed event, retried on 5xx and not retried + * after 410 or 413. + * + * Two shapes of run matter and both are here. A fixture that accepts an + * `http://` loopback callback delivers, which fails the two SSRF rows and makes + * every other row gradeable — the state a demo fixture with its private-network + * guard off is in. A fixture that refuses it passes the SSRF rows and reports + * the delivery rows untestable, which is what a hardened server does and what + * the second implementation the suite was run against actually did. + * + * Timings are stubbed down hard. `EVENTS_DELIVERY_WAIT_MS` bounds how long the + * scenario waits for a first POST and `EVENTS_DELIVERY_SETTLE_MS` how long it + * lets retries play out; at their defaults one run of this scenario is about + * twenty seconds of mostly sleeping, and even stubbed down it is about ten. + * + * So the cases run concurrently. Each builds its own fixture on its own port and + * touches no shared state, and the scenario is imported once in `beforeAll` + * rather than per case, which is what makes that safe: `vi.resetModules()` in + * the middle of a concurrent run would pull the module graph out from under a + * case already using it. One graph also means one set of timings for the file. + * + * As in the push controls, the run context has to come from the same fresh + * import as the scenario, or `instanceof JsonRpcError` is false across two + * copies of the class. + */ + +type Imported = { + Scenario: typeof import('./webhook-delivery').EventsWebhookDeliveryScenario; + testContext: typeof import('../../../connection/testing').testContext; + takeWireViolations: typeof import('../../../validation/wire-schema').takeWireViolations; +}; +let mod: Imported; + +beforeAll(async () => { + vi.stubEnv('EVENTS_DELIVERY_WAIT_MS', '1500'); + // Comfortably over the fixture's 1.1s retry spacing, which is itself over a + // second because `webhook-timestamp` is in whole seconds and two attempts + // inside one second would share a stamp the fixture did freshen. + vi.stubEnv('EVENTS_DELIVERY_SETTLE_MS', '2000'); + // Never inherit a real tunnel from the environment: these tests are about the + // loopback receiver, which is also the SSRF probe. + vi.stubEnv('EVENTS_WEBHOOK_CALLBACK_BASE', ''); + vi.resetModules(); + const [scenario, testing, wire] = await Promise.all([ + import('./webhook-delivery'), + import('../../../connection/testing'), + import('../../../validation/wire-schema') + ]); + mod = { + Scenario: scenario.EventsWebhookDeliveryScenario, + testContext: testing.testContext, + takeWireViolations: wire.takeWireViolations + }; +}); + +afterAll(() => { + vi.unstubAllEnvs(); +}); + +async function deliveryChecks(opts: EventsFixtureOptions) { + const fixture = await startEventsFixture(opts); + try { + const checks = await new mod.Scenario().run( + mod.testContext(fixture.url, DRAFT_PROTOCOL_VERSION) + ); + // Drained so an intentionally malformed delivery does not trip the global + // vitest hook; these tests assert on the check, not the wire validator. + mod.takeWireViolations(); + return new Map(checks.map((c) => [c.id, c])); + } finally { + await fixture.close(); + } +} + +/** + * A server that delivers to the loopback receiver. `acceptHttpUrl` is not a + * detail: the receiver is `http://127.0.0.1`, so a server that enforces https + * never delivers at all, and the delivery rows are only reachable through a + * fixture that has its guard off. + */ +function delivering(delivery: DeliveryBehaviour = {}): EventsFixtureOptions { + return { + capability: { listChanged: true }, + descriptors: [descriptor({ name: 'hook.event', delivery: ['webhook'] })], + subscribe: { acceptHttpUrl: true }, + // A source that fires more than once, like any real one, so the 410 probe + // can tell a suspended subscription from a quiet event. + delivery: { followUpEvents: 2, ...delivery } + }; +} + +/** Generous, because every case in the file is in flight at once. */ +const TIMEOUT = 60_000; + +describe.concurrent('a server that delivers to loopback', () => { + test( + 'every row but the SSRF pair grades, and the SSRF pair fails', + async () => { + const checks = await deliveryChecks(delivering()); + + // Delivered to 127.0.0.1, which is the whole point of the probe. + const validate = checks.get('sep-3415-ssrf-validate-callback-url'); + expect(validate?.status).toBe('FAILURE'); + expect(validate?.errorMessage).toContain('loopback'); + expect(checks.get('sep-3415-ssrf-reject-non-routable')?.status).toBe( + 'FAILURE' + ); + + for (const id of [ + 'sep-3415-verification-required-before-delivery', + 'sep-3415-verification-challenge-echo', + 'sep-3415-delivery-post-json', + 'sep-3415-delivery-standard-webhooks-headers', + 'sep-3415-delivery-subscription-id-header', + 'sep-3415-delivery-signature-formula', + 'sep-3415-delivery-body-size', + 'sep-3415-delivery-410-non-retryable', + 'sep-3415-delivery-413-non-retryable', + 'sep-3415-delivery-retry-regenerates-signature', + 'sep-3415-delivery-retries-bounded', + 'sep-3415-envelope-type-discriminator', + 'sep-3415-envelope-signed-like-deliveries', + 'sep-3415-envelope-webhook-id-format', + 'sep-3415-ssrf-no-redirects' + ]) { + expect(checks.get(id)?.status, id).toBe('SUCCESS'); + } + }, + TIMEOUT + ); + + // The shape a hardened server is in, and the shape the second implementation + // the suite was run against was in. + test( + 'refusing the loopback callback passes the SSRF rows and grades nothing else green', + async () => { + const checks = await deliveryChecks({ + capability: { listChanged: true }, + descriptors: [ + descriptor({ name: 'hook.event', delivery: ['webhook'] }) + ], + subscribe: { rejectNonRoutableUrl: true } + }); + expect(checks.get('sep-3415-ssrf-validate-callback-url')?.status).toBe( + 'SUCCESS' + ); + expect(checks.get('sep-3415-ssrf-reject-non-routable')?.status).toBe( + 'SUCCESS' + ); + const signature = checks.get('sep-3415-delivery-signature-formula'); + expect(signature?.details?.untestable).toBe(true); + expect(signature?.errorMessage).toContain('EVENTS_WEBHOOK_CALLBACK_BASE'); + // The refusal answers two SSRF rows and not the other two, which still + // have to be reported: a row that appears only on the delivering path + // makes the two runs' counts incomparable. + for (const id of [ + 'sep-3415-ssrf-no-redirects', + 'sep-3415-ssrf-validate-at-delivery-time' + ]) { + expect(checks.get(id), id).toBeDefined(); + expect(checks.get(id)?.details?.untestable, id).toBe(true); + } + }, + TIMEOUT + ); + + test( + 'accepting the callback and then never delivering fails nothing it cannot see', + async () => { + // Subscribe accepted, nothing POSTed: the SSRF row says so rather than + // claiming the server passed a check it was never put to. + const checks = await deliveryChecks({ + capability: { listChanged: true }, + descriptors: [ + descriptor({ name: 'hook.event', delivery: ['webhook'] }) + ], + subscribe: { acceptHttpUrl: true } + }); + const validate = checks.get('sep-3415-ssrf-validate-callback-url'); + expect(validate?.status).toBe('WARNING'); + expect(validate?.errorMessage).toContain('nothing was delivered'); + expect( + checks.get('sep-3415-ssrf-reject-non-routable')?.errorMessage + ).toContain('delivery-time validation'); + expect( + checks.get('sep-3415-delivery-post-json')?.details?.untestable + ).toBe(true); + }, + TIMEOUT + ); +}); + +describe.concurrent('reject-non-routable, apart from the scheme rule', () => { + const hook = [descriptor({ name: 'hook.event', delivery: ['webhook'] })]; + + // The scheme rule refuses the http loopback receiver on its own, so that + // refusal used to pass this row for a server that never checks routability. + test( + 'a server enforcing https alone fails when it dials the https loopback probe', + async () => { + const checks = await deliveryChecks({ + capability: { listChanged: true }, + descriptors: hook, + delivery: {} + }); + expect(checks.get('sep-3415-ssrf-validate-callback-url')?.status).toBe( + 'SUCCESS' + ); + const check = checks.get('sep-3415-ssrf-reject-non-routable'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('connected to it'); + }, + TIMEOUT + ); + + test( + 'an error returned after dialling the probe is not a refusal', + async () => { + const checks = await deliveryChecks({ + capability: { listChanged: true }, + descriptors: hook, + delivery: { synchronousVerification: true } + }); + const check = checks.get('sep-3415-ssrf-reject-non-routable'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('only after connecting'); + expect(check?.details?.code).toBe(-32027); + }, + TIMEOUT + ); + + test( + 'accepting the probe without ever dialling it warns', + async () => { + const checks = await deliveryChecks({ + capability: { listChanged: true }, + descriptors: hook + }); + const check = checks.get('sep-3415-ssrf-reject-non-routable'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('delivery-time validation'); + }, + TIMEOUT + ); + + test( + 'refusing the probe without dialling it passes', + async () => { + const checks = await deliveryChecks({ + capability: { listChanged: true }, + descriptors: hook, + subscribe: { rejectNonRoutableUrl: true }, + delivery: {} + }); + const check = checks.get('sep-3415-ssrf-reject-non-routable'); + expect(check?.status).toBe('SUCCESS'); + expect(check?.details?.connections).toBe(0); + }, + TIMEOUT + ); +}); + +describe.concurrent('the verification handshake', () => { + // The divergence this row exists for: kitchen-sink delivers with no handshake + // at all, because the handshake does not exist yet. + test( + 'delivering with no challenge fails, and says what it lets an attacker do', + async () => { + const checks = await deliveryChecks(delivering({ verify: false })); + const check = checks.get( + 'sep-3415-verification-required-before-delivery' + ); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('third party'); + expect( + checks.get('sep-3415-verification-challenge-echo')?.details?.untestable + ).toBe(true); + }, + TIMEOUT + ); + + // A server that verifies inside events/subscribe, the way the document's + // "returned synchronously from events/subscribe" reads, refuses a callback + // that echoes the wrong nonce with -32027 challenge_failed. + test( + 'refusing a wrong echo with -32027 challenge_failed passes the failure row', + async () => { + const checks = await deliveryChecks( + delivering({ synchronousVerification: true }) + ); + const check = checks.get('sep-3415-verification-failure-error'); + expect(check?.status).toBe('SUCCESS'); + }, + TIMEOUT + ); + + test( + 'accepting a subscription whose echo was wrong fails the failure row', + async () => { + const checks = await deliveryChecks( + delivering({ synchronousVerification: true, ignoreFailedEcho: true }) + ); + const check = checks.get('sep-3415-verification-failure-error'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('-32027'); + }, + TIMEOUT + ); + + // The default fixture challenges after subscribe returns and ignores the + // answer, so it delivers to an endpoint that never consented. + test( + 'delivering after a wrong echo fails the failure row', + async () => { + const checks = await deliveryChecks(delivering()); + const check = checks.get('sep-3415-verification-failure-error'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('delivered'); + }, + TIMEOUT + ); + + test( + 'with no challenge at all, the failure row stays untestable', + async () => { + const checks = await deliveryChecks(delivering({ verify: false })); + expect( + checks.get('sep-3415-verification-failure-error')?.details?.untestable + ).toBe(true); + }, + TIMEOUT + ); + + // The probe callbacks used to answer every POST with their failure status, + // the challenge included, so a server that verifies synchronously could not + // subscribe them and every probe row went untestable. + test( + 'a synchronous verifier still has its probe rows graded', + async () => { + const checks = await deliveryChecks( + delivering({ synchronousVerification: true }) + ); + for (const id of [ + 'sep-3415-delivery-410-non-retryable', + 'sep-3415-delivery-413-non-retryable', + 'sep-3415-delivery-retries-bounded', + 'sep-3415-ssrf-no-redirects' + ]) { + const check = checks.get(id); + expect( + check?.details?.untestable, + `${id}: ${check?.errorMessage}` + ).not.toBe(true); + expect(check?.status, `${id}: ${check?.errorMessage}`).toBe('SUCCESS'); + } + }, + TIMEOUT + ); + + test( + 'delivering an event before the challenge fails', + async () => { + const checks = await deliveryChecks( + delivering({ eventBeforeVerification: true }) + ); + const check = checks.get( + 'sep-3415-verification-required-before-delivery' + ); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('before the verification'); + }, + TIMEOUT + ); +}); + +describe.concurrent('signing', () => { + // The divergence this row exists for: kitchen-sink keys the HMAC on the + // literal `whsec_…` string, where the document says the key is the + // base64-decoded bytes after the prefix. No Standard Webhooks receiver + // verifies those signatures. + test( + 'keying the HMAC on the literal whsec_ string fails', + async () => { + const checks = await deliveryChecks( + delivering({ literalKeySignature: true }) + ); + const check = checks.get('sep-3415-delivery-signature-formula'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('did not verify over the raw body'); + // Freshness is graded separately, so a wrong formula must not be + // reported as reusing timestamps it plainly did freshen. + expect( + checks.get('sep-3415-delivery-retry-regenerates-signature')?.status + ).toBe('SUCCESS'); + }, + TIMEOUT + ); + + test( + 'omitting webhook-signature fails the header row and un-grades the formula', + async () => { + const checks = await deliveryChecks( + delivering({ omitHeaders: ['webhook-signature'] }) + ); + const headers = checks.get('sep-3415-delivery-standard-webhooks-headers'); + expect(headers?.status).toBe('FAILURE'); + expect(headers?.errorMessage).toContain('webhook-signature'); + expect( + checks.get('sep-3415-delivery-signature-formula')?.details?.untestable + ).toBe(true); + }, + TIMEOUT + ); + + test( + 'omitting the subscription id header fails, and says why it matters', + async () => { + const checks = await deliveryChecks( + delivering({ omitSubscriptionIdHeader: true }) + ); + const check = checks.get('sep-3415-delivery-subscription-id-header'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('pick a secret'); + }, + TIMEOUT + ); + + test( + 'a subscription id header for another subscription fails', + async () => { + const checks = await deliveryChecks( + delivering({ wrongSubscriptionIdHeader: true }) + ); + const check = checks.get('sep-3415-delivery-subscription-id-header'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('other than the subscribe'); + }, + TIMEOUT + ); +}); + +describe.concurrent('transport and body', () => { + test( + 'delivering as text/plain fails the POST-and-JSON row', + async () => { + const checks = await deliveryChecks( + delivering({ contentType: 'text/plain' }) + ); + const check = checks.get('sep-3415-delivery-post-json'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('text/plain'); + }, + TIMEOUT + ); + + test( + 'a body past 256 KiB warns rather than fails', + async () => { + const checks = await deliveryChecks(delivering({ oversizedBody: true })); + const check = checks.get('sep-3415-delivery-body-size'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('256 KiB'); + }, + TIMEOUT + ); +}); + +describe.concurrent('retries and redirects', () => { + test( + 'reusing the timestamp across retries fails the freshness row', + async () => { + const checks = await deliveryChecks( + delivering({ staleRetrySignature: true }) + ); + const check = checks.get('sep-3415-delivery-retry-regenerates-signature'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('freshness window'); + }, + TIMEOUT + ); + + // webhook-timestamp is in whole seconds, so two attempts inside one second + // carry the same stamp whether or not the server regenerated it. mcpkit's + // first retry is 500ms out, which made this row flap on it. + test( + 'sub-second retries sharing a stamp do not fail the freshness row', + async () => { + const checks = await deliveryChecks(delivering({ retryGapMs: 300 })); + const check = checks.get('sep-3415-delivery-retry-regenerates-signature'); + expect(check?.status, check?.errorMessage).not.toBe('FAILURE'); + }, + TIMEOUT + ); + + test( + 'retrying after 410 and 413 fails both non-retryable rows', + async () => { + const checks = await deliveryChecks( + delivering({ retryNonRetryable: true, attempts: 3 }) + ); + const gone = checks.get('sep-3415-delivery-410-non-retryable'); + expect(gone?.status).toBe('FAILURE'); + expect(gone?.errorMessage).toContain('non-retryable'); + expect(checks.get('sep-3415-delivery-413-non-retryable')?.status).toBe( + 'FAILURE' + ); + }, + TIMEOUT + ); + + test( + 'suspending the subscription after a 410 fails the 410 row alone', + async () => { + const checks = await deliveryChecks(delivering({ stopAfterGone: true })); + const gone = checks.get('sep-3415-delivery-410-non-retryable'); + expect(gone?.status).toBe('FAILURE'); + expect(gone?.errorMessage).toContain('must not suspend'); + expect(checks.get('sep-3415-delivery-413-non-retryable')?.status).toBe( + 'SUCCESS' + ); + }, + TIMEOUT + ); + + test( + 'a source that fires once leaves 410 survival untestable, not green', + async () => { + const checks = await deliveryChecks(delivering({ followUpEvents: 0 })); + const gone = checks.get('sep-3415-delivery-410-non-retryable'); + expect(gone?.details?.untestable).toBe(true); + expect(gone?.status).toBe('WARNING'); + }, + TIMEOUT + ); + + test( + 'a categorical lastError on the refresh after a 413 passes', + async () => { + const checks = await deliveryChecks(delivering({ deliveryStatus: true })); + const row = checks.get('sep-3415-delivery-status-last-error-category'); + expect(row?.status, row?.errorMessage).toBe('SUCCESS'); + expect(row?.details?.lastError).toBe('http_4xx'); + }, + TIMEOUT + ); + + test( + 'a lastError carrying the endpoint response fails', + async () => { + const checks = await deliveryChecks( + delivering({ + deliveryStatus: { + lastError: '413 Payload Too Large: nope' + } + }) + ); + const row = checks.get('sep-3415-delivery-status-last-error-category'); + expect(row?.status).toBe('FAILURE'); + expect(row?.errorMessage).toContain('not one of'); + }, + TIMEOUT + ); + + test( + 'no deliveryStatus leaves lastError untestable', + async () => { + const checks = await deliveryChecks(delivering()); + const row = checks.get('sep-3415-delivery-status-last-error-category'); + expect(row?.details?.untestable).toBe(true); + }, + TIMEOUT + ); + + test( + 'following a 302 fails the no-redirects row', + async () => { + const checks = await deliveryChecks( + delivering({ followRedirects: true }) + ); + const check = checks.get('sep-3415-ssrf-no-redirects'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('followed a 302'); + }, + TIMEOUT + ); +}); + +describe.concurrent('control envelopes', () => { + test( + 'an unsigned envelope fails, since a receiver cannot verify it', + async () => { + const checks = await deliveryChecks(delivering({ signEnvelopes: false })); + const check = checks.get('sep-3415-envelope-signed-like-deliveries'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('cannot verify it'); + }, + TIMEOUT + ); + + test( + 'a webhook-id outside msg__ warns', + async () => { + const checks = await deliveryChecks( + delivering({ envelopeIdFormat: 'plain' }) + ); + const check = checks.get('sep-3415-envelope-webhook-id-format'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('outside the documented form'); + }, + TIMEOUT + ); + + // These two normally report untestable, because no client can ask a server + // for a retention gap or a revocation. A server that sends one unasked is + // graded on it, which is the only evidence these checks work. + test( + 'gap and terminated envelopes are graded when they arrive', + async () => { + const quiet = await deliveryChecks(delivering()); + expect(quiet.get('sep-3415-envelope-gap')?.details?.untestable).toBe( + true + ); + expect( + quiet.get('sep-3415-envelope-terminated')?.details?.untestable + ).toBe(true); + + const sent = await deliveryChecks( + delivering({ gapEnvelope: true, terminatedEnvelope: true }) + ); + expect(sent.get('sep-3415-envelope-gap')?.status).toBe('SUCCESS'); + expect(sent.get('sep-3415-envelope-terminated')?.status).toBe('SUCCESS'); + }, + TIMEOUT * 2 + ); + + test( + 'a gap envelope with no fresh cursor warns', + async () => { + const checks = await deliveryChecks( + delivering({ gapEnvelope: { cursor: null } }) + ); + const check = checks.get('sep-3415-envelope-gap'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('nothing to persist'); + }, + TIMEOUT + ); +}); + +describe.concurrent( + 'envelopes a control asks for, on this subscription', + () => { + const withControls = ( + controls: EventsFixtureOptions['webhookEnvelopeControls'] + ): EventsFixtureOptions => ({ + ...delivering(), + webhookEnvelopeControls: controls + }); + + test( + 'a gap and a termination on request both grade, and count as signed envelopes', + async () => { + const checks = await deliveryChecks( + withControls({ gap: true, terminate: true }) + ); + expect(checks.get('sep-3415-envelope-gap')?.status).toBe('SUCCESS'); + expect(checks.get('sep-3415-envelope-terminated')?.status).toBe( + 'SUCCESS' + ); + expect( + checks.get('sep-3415-envelope-signed-like-deliveries')?.status + ).toBe('SUCCESS'); + }, + TIMEOUT + ); + + test( + 'envelopes missing their cursor or error warn', + async () => { + const checks = await deliveryChecks( + withControls({ gap: { cursor: null }, terminate: { error: null } }) + ); + // Graded from the envelope that arrived, not reported untestable, which + // is also a WARNING and would pass the status check on its own. + const gap = checks.get('sep-3415-envelope-gap'); + expect(gap?.status).toBe('WARNING'); + expect(gap?.details?.body).toMatchObject({ type: 'gap', cursor: null }); + const terminated = checks.get('sep-3415-envelope-terminated'); + expect(terminated?.status).toBe('WARNING'); + expect(terminated?.details?.body).toMatchObject({ + type: 'terminated', + error: null + }); + }, + TIMEOUT + ); + + test( + 'a control that acknowledges and sends nothing is graded, not untestable', + async () => { + const checks = await deliveryChecks( + withControls({ gap: 'silent', terminate: 'silent' }) + ); + const gap = checks.get('sep-3415-envelope-gap'); + expect(gap?.status).toBe('WARNING'); + expect(gap?.details?.untestable).toBeUndefined(); + const terminated = checks.get('sep-3415-envelope-terminated'); + expect(terminated?.status).toBe('FAILURE'); + expect(terminated?.details?.untestable).toBeUndefined(); + expect(terminated?.errorMessage).toContain('never told'); + }, + TIMEOUT + ); + + test( + 'without the controls both rows stay untestable and name them', + async () => { + const checks = await deliveryChecks(delivering()); + const gap = checks.get('sep-3415-envelope-gap'); + expect(gap?.details?.untestable).toBe(true); + expect(gap?.errorMessage).toContain('events_conformance_webhook_gap'); + const terminated = checks.get('sep-3415-envelope-terminated'); + expect(terminated?.details?.untestable).toBe(true); + expect(terminated?.errorMessage).toContain( + 'events_conformance_webhook_terminate' + ); + }, + TIMEOUT + ); + } +); + +describe.concurrent('confirming intent without a handshake', () => { + // The document allows four consent paths and the harness can only see two of + // them. Before this, a server that read the receiver's well-known document and + // delivered without a challenge failed the rule it had just satisfied. + test( + 'a server that reads the well-known document passes without challenging', + async () => { + const checks = await deliveryChecks( + delivering({ verifyViaWellKnown: true, verify: false }) + ); + const check = checks.get( + 'sep-3415-verification-required-before-delivery' + ); + expect(check?.status).toBe('SUCCESS'); + expect(check?.details?.via).toBe( + '/.well-known/mcp-webhook-receiver.json' + ); + // The challenge rows say why they were not exercised, and how to force it. + const echo = checks.get('sep-3415-verification-challenge-echo'); + expect(echo?.details?.untestable).toBe(true); + expect(echo?.errorMessage).toContain('EVENTS_RECEIVER_WELL_KNOWN=0'); + }, + TIMEOUT + ); + + test( + 'delivering with neither a challenge nor a document read still fails', + async () => { + const checks = await deliveryChecks(delivering({ verify: false })); + const check = checks.get( + 'sep-3415-verification-required-before-delivery' + ); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('third party'); + }, + TIMEOUT + ); +}); + +describe.concurrent('the -32027 error-table row', () => { + test( + 'a categorised challenge failure passes, and an uncategorised one warns', + async () => { + const ok = await deliveryChecks( + delivering({ synchronousVerification: true }) + ); + const good = ok.get('sep-3415-error-callback-endpoint-error'); + expect(good?.status).toBe('SUCCESS'); + expect(good?.details?.reason).toBe('challenge_failed'); + + const vague = await deliveryChecks( + delivering({ + synchronousVerification: true, + challengeFailureReason: 'it did not work' + }) + ); + const check = vague.get('sep-3415-error-callback-endpoint-error'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('not one of the documented'); + }, + TIMEOUT * 2 + ); + + test( + 'the wrong code for a failed callback fails the row', + async () => { + const checks = await deliveryChecks( + delivering({ + synchronousVerification: true, + challengeFailureCode: -32603 + }) + ); + const check = checks.get('sep-3415-error-callback-endpoint-error'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('-32027 CallbackEndpointError'); + }, + TIMEOUT + ); + + test( + 'a run where no callback ever fails reports the row untestable', + async () => { + const checks = await deliveryChecks(delivering()); + const check = checks.get('sep-3415-error-callback-endpoint-error'); + expect(check?.details?.untestable).toBe(true); + expect(check?.errorMessage).toContain('never provoked'); + }, + TIMEOUT + ); +}); + +describe.concurrent('leaking the endpoint response', () => { + // Retires an unconditional SUCCESS: the row used to pass on a run where + // nothing had failed, so it could never have caught a leak. + test( + 'echoing the endpoint body back in the error fails', + async () => { + const checks = await deliveryChecks( + delivering({ + synchronousVerification: true, + challengeFailureLeaksBody: true + }) + ); + const check = checks.get( + 'sep-3415-verification-no-raw-endpoint-responses' + ); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('reflection'); + }, + TIMEOUT + ); + + test( + 'reporting only the category passes, and a quiet run is untestable', + async () => { + const ok = await deliveryChecks( + delivering({ synchronousVerification: true }) + ); + expect( + ok.get('sep-3415-verification-no-raw-endpoint-responses')?.status + ).toBe('SUCCESS'); + + const quiet = await deliveryChecks(delivering()); + const check = quiet.get( + 'sep-3415-verification-no-raw-endpoint-responses' + ); + expect(check?.details?.untestable).toBe(true); + expect(check?.status).toBe('WARNING'); + }, + TIMEOUT * 2 + ); +}); + +describe.concurrent( + 'a hardened server the harness can still be delivered to', + () => { + // The harness listens on loopback and a hardened server refuses a loopback + // callback, correctly. Both facts cannot hold in one subscription, so without + // an override the delivery rows are unreachable on any server that enforces + // the rule the SSRF rows grade. + test( + 'the SSRF rows are graded first, then the guard is lifted for one origin', + async () => { + const checks = await deliveryChecks({ + capability: { listChanged: true }, + descriptors: [ + descriptor({ name: 'hook.event', delivery: ['webhook'] }) + ], + subscribe: { + allowCallbackOriginControl: true, + rejectNonRoutableUrl: true + }, + delivery: {} + }); + + // Graded against the default configuration, before anything was lifted. + expect(checks.get('sep-3415-ssrf-validate-callback-url')?.status).toBe( + 'SUCCESS' + ); + expect(checks.get('sep-3415-ssrf-reject-non-routable')?.status).toBe( + 'SUCCESS' + ); + + // And the behavioural rows now grade, which is the whole point. + for (const id of [ + 'sep-3415-delivery-post-json', + 'sep-3415-delivery-standard-webhooks-headers', + 'sep-3415-delivery-signature-formula', + 'sep-3415-verification-required-before-delivery', + 'sep-3415-envelope-type-discriminator' + ]) { + expect(checks.get(id)?.status, id).toBe('SUCCESS'); + } + + // A delivery to a deliberately-permitted origin proves nothing about + // delivery-time revalidation, so that row says so rather than passing. + const rebind = checks.get('sep-3415-ssrf-validate-at-delivery-time'); + expect(rebind?.status).toBe('SKIPPED'); + expect(rebind?.errorMessage).toContain('permitted through'); + }, + TIMEOUT * 2 + ); + + test( + 'without the control the rows stay untestable and name both ways out', + async () => { + const checks = await deliveryChecks({ + capability: { listChanged: true }, + descriptors: [ + descriptor({ name: 'hook.event', delivery: ['webhook'] }) + ], + subscribe: {}, + delivery: {} + }); + expect(checks.get('sep-3415-ssrf-validate-callback-url')?.status).toBe( + 'SUCCESS' + ); + const check = checks.get('sep-3415-delivery-signature-formula'); + expect(check?.details?.untestable).toBe(true); + expect(check?.errorMessage).toContain('EVENTS_WEBHOOK_CALLBACK_BASE'); + expect(check?.errorMessage).toContain( + 'events_conformance_allow_callback_origin' + ); + }, + TIMEOUT + ); + } +); diff --git a/src/scenarios/server/events/negative-fixture.ts b/src/scenarios/server/events/negative-fixture.ts new file mode 100644 index 00000000..62700c77 --- /dev/null +++ b/src/scenarios/server/events/negative-fixture.ts @@ -0,0 +1,1734 @@ +/** + * The fixture the MCP Events negative controls are graded against. + * + * One server, five scenarios. Every option here exists to break exactly one + * rule while leaving the rest of the wire conformant, which is what makes a + * negative control a control: if two things are wrong at once, a flipped check + * does not say which one it caught. + * + * It speaks the SEP-2575 stateless wire (`server/discover` plus one POST per + * request), built per test rather than checked in as an example server, which + * matches the SEP-2640 negative tests. `events/stream` is a real SSE response + * held open with timers rather than a canned transcript, because the heartbeat + * cadence and the cancellation rows are about timing. + */ + +import { EVENTS_EXTENSION_ID } from './helpers'; +import { createHash, createHmac } from 'crypto'; +import { createServer, type IncomingMessage, type ServerResponse } from 'http'; +import type { AddressInfo } from 'net'; +import { withRequiredDraftResultFields } from '../../../mock-server'; +import { DRAFT_PROTOCOL_VERSION } from '../../../types'; +import { SUBSCRIPTION_ID_META } from './helpers'; + +/** A descriptor that is well formed apart from whatever a test overrides. */ +export function descriptor(overrides: Record = {}) { + return { + name: 'test.event', + description: 'A negative-control fixture event type.', + delivery: ['poll'], + inputSchema: { + type: 'object', + properties: { channel: { type: 'string' } } + }, + payloadSchema: { type: 'object', properties: { id: { type: 'string' } } }, + ...overrides + }; +} + +/** A poll result that is well formed apart from whatever a test overrides. */ +export function pollResult(overrides: Record = {}) { + return { + events: [], + cursor: 'cursor_001', + truncated: false, + hasMore: false, + nextPollMs: 30000, + ...overrides + }; +} + +/** An occurrence that is well formed apart from whatever a test overrides. */ +export function occurrence(overrides: Record = {}) { + return { + eventId: 'evt_001', + name: 'test.event', + timestamp: '2026-09-15T12:00:00Z', + data: { id: 'x' }, + ...overrides + }; +} + +export type JsonRpcErrorShape = { code: number; message: string }; + +/** A queued poll answer: a result body, or an error to answer with instead. */ +export type PollAnswer = Record | { error: JsonRpcErrorShape }; + +/** + * How `events/stream` behaves. The defaults are conformant: an immediate + * `active` confirmation carrying the parent request id in `_meta`, a heartbeat + * with a cursor, and one well-formed event. + */ +export interface StreamBehaviour { + /** Answer the POST with this JSON-RPC error instead of opening a stream. */ + error?: JsonRpcErrorShape; + /** Stream a type whose `delivery` omits `push`, instead of refusing it. */ + acceptNonPushType?: boolean; + /** Answer the POST with a plain JSON result, which is not a stream at all. */ + answerJson?: boolean; + /** Send no `notifications/events/active`. */ + omitActive?: boolean; + /** Fields merged into the `active` params. */ + activeParams?: Record; + /** + * Where the parent request id goes. `meta` is the spelling the document + * requires; `requestId` is the one kitchen-sink ships; `none` omits it. + */ + correlation?: 'meta' | 'requestId' | 'none'; + /** Heartbeat cadence. 0 sends none. */ + heartbeatMs?: number; + /** Fields merged into each heartbeat's params. */ + heartbeatParams?: Record; + /** Send `: keepalive` SSE comment lines at the heartbeat cadence too. */ + sseComments?: boolean; + /** Deliver an event this long after the stream opens. 0 sends none. */ + eventAfterMs?: number; + /** Fields merged into the delivered event's params. */ + eventParams?: Record; + /** Ride a non-`notifications/events/*` notification on the stream. */ + foreignNotification?: string; + /** Send `notifications/events/error` after this long. */ + errorNotificationAfterMs?: number; + /** Send `notifications/events/terminated` after this long. */ + terminatedAfterMs?: number; + /** Send a second `active` with `truncated: true`, the retention-gap signal. */ + gapAfterMs?: number; + /** Close the stream from the server side, writing a final frame first. */ + closeAfterMs?: number; + /** The `result` of that final frame. Defaults to an empty typed result. */ + finalResult?: Record; + /** Refuse concurrent opens past this many, the cap streams are exempt from. */ + maxConcurrent?: number; + /** Open a stream for any name, including one the catalog does not serve. */ + acceptAnyName?: boolean; + /** Name the quota in `data.limit` when refusing past `maxConcurrent`. */ + capLimitName?: string; +} + +const CONFORMANT_STREAM: Required< + Pick< + StreamBehaviour, + 'correlation' | 'heartbeatMs' | 'eventAfterMs' | 'maxConcurrent' + > +> = { + correlation: 'meta', + heartbeatMs: 150, + eventAfterMs: 80, + maxConcurrent: Infinity +}; + +/** + * How `events/subscribe` and `events/unsubscribe` behave. The defaults are + * conformant: an https-only callback, a validated `whsec_` secret, an id + * derived from the whole key, an idempotent upsert, and `-32023 NotFound` for + * a key the server does not hold. + */ +export interface SubscribeBehaviour { + /** Answer every subscribe with this error instead of subscribing. */ + error?: JsonRpcErrorShape; + /** Accept a subscribe carrying no `delivery.secret`. */ + acceptMissingSecret?: boolean; + /** Accept a secret without the `whsec_` prefix. */ + acceptBadPrefix?: boolean; + /** Accept a `whsec_` secret that decodes to fewer than 24 bytes. */ + acceptShortSecret?: boolean; + /** Accept an `http://` callback URL. */ + acceptHttpUrl?: boolean; + /** + * Refuse a callback whose host is a loopback, private, link-local or + * unspecified literal, with the rejection code. Off by default so the + * default fixture enforces the scheme rule alone, which is the state that + * used to pass `reject-non-routable` on the scheme refusal. + */ + rejectNonRoutableUrl?: boolean; + /** Code for a rejected secret or URL, where the document says -32602. */ + rejectionCode?: number; + /** Subscribe to a type whose `delivery` does not list `webhook`. */ + acceptNonWebhookType?: boolean; + /** Grant `refreshBefore: null` however finite the suggestion. */ + nullRefreshAlways?: boolean; + /** Grant `refreshBefore: null` when `ttlMs` was omitted. */ + nullRefreshOnOmitted?: boolean; + /** Grant a `refreshBefore` well past the suggestion. */ + grantBeyondSuggestion?: boolean; + /** Raw `refreshBefore` to answer with, for the non-timestamp case. */ + refreshBefore?: unknown; + /** Leave `refreshBefore` out of the subscribe result entirely. */ + omitRefreshBefore?: boolean; + /** Fields merged into a successful unsubscribe result, which is otherwise empty. */ + unsubscribeResult?: Record; + /** Reject a `ttlMs` the server dislikes instead of clamping it. */ + rejectTtl?: 'short' | 'long' | 'both'; + /** Fields merged into every subscribe result. */ + result?: Record; + /** Omit `id` from the subscribe result. */ + omitId?: boolean; + /** Mint a fresh id per call, so a repeat subscribe is not an upsert. */ + nonIdempotentId?: boolean; + /** Derive the id from `(name, arguments)` only, leaving the URL out. */ + idIgnoresUrl?: boolean; + /** Treat a supplied `id` as addressing that subscription. */ + idIsAnInput?: boolean; + /** Answer success for an unsubscribe of a key never held. */ + unsubscribeUnknownOk?: boolean; + /** Code for an unsubscribe of an unknown key, where the document says -32023. */ + unsubscribeUnknownCode?: number; + /** Code for an unsubscribe of a key the server does hold. */ + unsubscribeHeldCode?: number; + /** Answer -32025 once this many subscriptions are live. */ + maxSubscriptions?: number; + /** + * Expose `events_conformance_allow_callback_origin` as a tool, which permits + * one origin past the callback guards for the rest of the process. + * + * There is no separate "harden" switch: leaving `acceptHttpUrl` unset already + * refuses the harness's loopback http callback, which is what a server with + * its SSRF guards on does. + */ + allowCallbackOriginControl?: boolean; +} + +/** + * What the fixture POSTs to a webhook callback once a subscription exists. The + * defaults are conformant: a verification challenge first, then one event, both + * signed per Standard Webhooks with the decoded secret bytes, both carrying + * `X-MCP-Subscription-Id`, retried on 5xx with a fresh timestamp and signature, + * and never retried after 410 or 413. + * + * Delivering at all is what a *loopback* callback makes wrong, so the SSRF rows + * fail whenever this is switched on without `EVENTS_WEBHOOK_CALLBACK_BASE`. + * That is the fixture standing in for a server run with its private-network + * guard disabled, which is the state the demo fixtures ship in. + */ +export interface DeliveryBehaviour { + /** Events to send after the first, each with a fresh `webhook-id`. */ + followUpEvents?: number; + /** Stop sending to a callback once it has answered `410`, as a server that suspends on it would. */ + stopAfterGone?: boolean; + /** + * Report `deliveryStatus` on a refresh, `lastError` being the category of the + * callback's last failed response. A string replaces the category, for the + * defect where raw endpoint detail leaks into it. + */ + deliveryStatus?: true | { lastError: string }; + /** Send the verification envelope before any event. */ + verify?: boolean; + /** + * Run the handshake inside `events/subscribe`, before answering, and refuse + * with -32027 `challenge_failed` when the callback does not echo the nonce + * in a 2xx body. The default challenges after the response and ignores the + * answer. + */ + synchronousVerification?: boolean; + /** With `synchronousVerification`, accept the subscription anyway. */ + ignoreFailedEcho?: boolean; + /** + * Confirm intent by fetching the callback origin's + * `/.well-known/mcp-webhook-receiver.json` instead of running the handshake, + * which is the fourth consent path the document allows. A document covering + * the callback path means no challenge POST; a 404 means no consent, and this + * fixture then delivers anyway, which is the case the rule exists to catch. + */ + verifyViaWellKnown?: boolean; + /** Code to answer when the handshake fails, where the table says -32027. */ + challengeFailureCode?: number; + /** `data.reason` for that failure, where the table wants a category. */ + challengeFailureReason?: string; + /** Echo the endpoint's own response body back in the failure message. */ + challengeFailureLeaksBody?: boolean; + /** Deliver an event at all. */ + sendEvent?: boolean; + /** Deliver the event before the verification envelope. */ + eventBeforeVerification?: boolean; + /** Sign with the literal `whsec_…` string instead of its decoded bytes. */ + literalKeySignature?: boolean; + /** Standard Webhooks headers to leave off. */ + omitHeaders?: string[]; + /** Leave off `X-MCP-Subscription-Id`. */ + omitSubscriptionIdHeader?: boolean; + /** Send an `X-MCP-Subscription-Id` that is not the subscribe response's id. */ + wrongSubscriptionIdHeader?: boolean; + /** Deliver with this method instead of POST. */ + method?: string; + /** Deliver with this Content-Type instead of application/json. */ + contentType?: string; + /** Pad the event body past the 256 KiB ceiling. */ + oversizedBody?: boolean; + /** Follow a 302 from the callback, which the document forbids. */ + followRedirects?: boolean; + /** Total attempts for a delivery the callback rejects with 5xx. */ + attempts?: number; + /** Milliseconds between retry attempts; defaults to just over a second. */ + retryGapMs?: number; + /** Reuse the first attempt's timestamp and signature on every retry. */ + staleRetrySignature?: boolean; + /** Retry after 410 and 413, which the document defines as non-retryable. */ + retryNonRetryable?: boolean; + /** Sign control envelopes the way deliveries are signed. */ + signEnvelopes?: boolean; + /** `webhook-id` form for control envelopes. */ + envelopeIdFormat?: 'msg' | 'plain'; + /** Send a `gap` control envelope. */ + gapEnvelope?: boolean | { cursor?: unknown }; + /** Send a `terminated` control envelope. */ + terminatedEnvelope?: boolean | { error?: unknown }; +} + +/** + * The restart, generation and subscription-state controls, for the TTL + * durability rows. Defaults are conformant: both grants survive a restart and + * nothing is garbage-collected. + */ +export interface DurabilityBehaviour { + /** Drop a no-expiry subscription this long after it was created. */ + gcAfterMs?: number; + /** Lose no-expiry subscriptions across a restart. */ + dropNoExpiryOnRestart?: boolean; + /** Lose finite subscriptions across a restart. */ + dropFiniteOnRestart?: boolean; + /** Answer the restart control without restarting. */ + restartNoop?: boolean; +} + +/** + * A per-event-type subscription cap on streams, the shape kitchen-sink's + * `Quota` has, reported through `events_conformance_quota`. + */ +export interface QuotaBehaviour { + /** Event type the cap applies to. */ + name: string; + /** Streams allowed at once on that type. */ + max: number; + /** Refuse past `max`. Off stands for a server that reports a cap it does not enforce. */ + enforce?: boolean; + /** `data.limit` on the refusal; null omits it. Defaults to `subscriptions`. */ + limitName?: string | null; + /** Expose `events_conformance_quota`. Defaults to true. */ + control?: boolean; +} + +/** + * `events_conformance_webhook_gap` / `_terminate`, which signal one webhook + * subscription by id. `true` sends the conformant envelope, an object + * overrides its fields, and `'silent'` acknowledges and sends nothing. + */ +export interface WebhookEnvelopeControls { + gap?: true | { cursor?: unknown } | 'silent'; + terminate?: true | { error?: unknown } | 'silent'; +} + +/** + * A second principal, through `events_conformance_subscribe_as` and + * `events_conformance_subscription_exists`, for the cross-tenant row. Defaults + * are conformant: the principal is part of the key, and unsubscribing one + * tenant's subscription leaves the other's alone. + */ +export interface TenantBehaviour { + /** Leave the principal out of the key, so both tenants share one subscription. */ + keyIgnoresPrincipal?: boolean; + /** Unsubscribing a key removes every tenant's subscription with that key. */ + unsubscribeIgnoresPrincipal?: boolean; +} + +/** + * `subscriptions/listen` and the catalog control. The conformant fixture + * echoes `eventsListChanged` exactly when the client asked and the declared + * capability says `listChanged: true`, and sends `notifications/events/list_changed` + * to every listener it echoed for whenever `events_conformance_catalog` adds or + * removes a type. + */ +export interface ListenBehaviour { + /** + * Which listeners get `eventsListChanged` echoed. `honor` is conformant; + * `always` echoes it even when the capability did not declare + * `listChanged: true` (breaks sep-3415-capability-list-changed-gated); + * `never` declines it even when it was declared, which leaves the + * notification with no channel (breaks sep-3415-list-changed-notification). + */ + echo?: 'honor' | 'always' | 'never'; + /** + * Set false to acknowledge the opt-in but never send the notification + * (breaks sep-3415-list-changed-notification). + */ + sendListChanged?: boolean; +} + +/** + * What `events_conformance_catalog` does to streams on a type it changes. The + * defaults are conformant: remove ends them with -32023 {kind: "event"}, + * retype with -32026 {feature: "inputSchema", reason: "schema_changed"}, and + * extend leaves them open. + */ +export interface CatalogBehaviour { + /** Remove without ending the streams on the type. */ + removeSilently?: boolean; + /** Retype without ending the streams on the type. */ + retypeSilently?: boolean; + /** End streams on a removed type with this error instead. */ + removeError?: { code: number; data?: unknown }; + /** End streams on a retyped type with this error instead. */ + retypeError?: { code: number; data?: unknown }; + /** End streams on an extended type, which an additive change must not do. */ + extendTerminates?: boolean; +} + +export interface EventsFixtureOptions { + /** Serve `subscriptions/listen` and expose `events_conformance_catalog`. */ + listen?: ListenBehaviour; + /** Expose `events_conformance_catalog` and set what its changes do to streams. */ + catalog?: CatalogBehaviour; + /** Expose the durability controls as tools. */ + durability?: DurabilityBehaviour; + /** Expose the two-tenant controls. */ + tenants?: TenantBehaviour; + /** Expose the per-subscription webhook envelope controls. */ + webhookEnvelopeControls?: WebhookEnvelopeControls; + /** + * One event type without replay: its streams confirm with `cursor: null`, + * and `events_conformance_yield_gap` is exposed. For that type the control + * sends `active{cursor: null, truncated: true}` when `truncatedOnGap` is set + * (the defect) and nothing otherwise; for any other type it sends the + * conformant fresh `active{cursor, truncated: true}`. + */ + noReplay?: { name: string; truncatedOnGap?: boolean }; + /** A per-type cap on streams, and optionally the control that reports it. */ + quota?: QuotaBehaviour; + /** + * Raw value to declare at `capabilities.extensions["io.modelcontextprotocol/events"]`; + * omit for no declaration. + */ + capability?: unknown; + descriptors?: object[]; + /** Answer `events/list` with this JSON-RPC error instead of a result. */ + listError?: JsonRpcErrorShape; + /** + * Poll responses, consumed in order; the last one repeats once exhausted. + * A `{ error }` entry makes that poll answer with a JSON-RPC error. + */ + pollResponses?: PollAnswer[]; + /** Overrides keyed by the polled event name, taking priority over the queue. */ + pollByName?: Record; + /** Error code for a poll naming an event type the fixture does not serve. */ + unknownNameCode?: number; + /** Error code for a poll whose arguments violate `inputSchema`. */ + invalidArgsCode?: number; + /** + * Leave `resultType` off every `events/*` result, including the stream's + * final frame. Breaks only sep-3415-result-type-complete. + */ + omitResultType?: boolean; + /** How `events/stream` behaves. */ + stream?: StreamBehaviour; + /** How `events/subscribe` and `events/unsubscribe` behave. */ + subscribe?: SubscribeBehaviour; + /** + * What the fixture POSTs to the callback. Omit it and the fixture subscribes + * without ever delivering, which is what a server with no webhook delivery + * looks like from the receiver's side. + */ + delivery?: DeliveryBehaviour; +} + +export interface EventsFixture { + url: string; + /** Params of every `events/poll` received, in order. */ + polls: Array>; + /** Params of every `events/stream` received, in order. */ + streams: Array>; + /** Params of every `events/subscribe` received, in order. */ + subscribes: Array>; + /** Subscription keys the fixture still holds when it is asked. */ + liveSubscriptions(): string[]; + close(): Promise; +} + +/** `whsec_` plus base64 of 24–64 bytes, which is what the document requires. */ +function secretProblem( + secret: unknown, + behaviour: SubscribeBehaviour +): 'missing' | 'prefix' | 'length' | undefined { + if (secret === undefined || secret === null) { + return behaviour.acceptMissingSecret ? undefined : 'missing'; + } + if (typeof secret !== 'string' || !secret.startsWith('whsec_')) { + return behaviour.acceptBadPrefix ? undefined : 'prefix'; + } + const bytes = Buffer.from(secret.slice('whsec_'.length), 'base64'); + if (bytes.length < 24 || bytes.length > 64) { + return behaviour.acceptShortSecret ? undefined : 'length'; + } + return undefined; +} + +export async function startEventsFixture( + opts: EventsFixtureOptions +): Promise { + const polls: Array> = []; + const streams: Array> = []; + const subscribes: Array> = []; + /** Live subscriptions, keyed the way the document keys them. */ + const subscriptions = new Map< + string, + { + id: string; + noExpiry?: boolean; + at?: number; + url?: string; + secret?: unknown; + name?: string; + } + >(); + let generation = 1; + /** The last non-2xx status each callback URL answered, cleared by a 2xx. */ + const lastFailureByUrl = new Map(); + /** `deliveryStatus` for a refresh, when the fixture reports it at all. */ + const deliveryStatusFor = ( + refreshing: boolean, + url: unknown + ): { deliveryStatus?: object } => { + const report = opts.delivery?.deliveryStatus; + const failed = + typeof url === 'string' ? lastFailureByUrl.get(url) : undefined; + if (!report || !refreshing || failed === undefined) return {}; + const lastError = + typeof report === 'object' + ? report.lastError + : failed >= 500 + ? 'http_5xx' + : 'http_4xx'; + return { deliveryStatus: { active: true, lastError } }; + }; + /** Deliveries still in flight, so close() can settle rather than abandon. */ + const inFlight = new Set>(); + /** Origins the callback guard has been asked to permit. */ + const permittedOrigins = new Set(); + let mintedIds = 0; + const queue = [...(opts.pollResponses ?? [pollResult()])]; + const descriptors = opts.descriptors ?? [descriptor()]; + const names = new Set( + descriptors + .map((d) => (d as { name?: unknown }).name) + .filter((n): n is string => typeof n === 'string') + ); + + /** Open streams, so close() can tear them down instead of hanging on them. */ + const openStreams = new Set<{ + res: ServerResponse; + stop: () => void; + name: string; + notify: (method: string, params: Record) => void; + }>(); + let liveStreams = 0; + const liveByName = new Map(); + + /** Open `subscriptions/listen` streams, and whether each was honored. */ + const listeners = new Set<{ + res: ServerResponse; + id: unknown; + echoed: boolean; + }>(); + let catalogMinted = 0; + const notifyListChanged = () => { + if (opts.listen?.sendListChanged === false) return; + for (const l of listeners) { + if (!l.echoed) continue; + l.res.write( + `data: ${JSON.stringify({ + jsonrpc: '2.0', + method: 'notifications/events/list_changed', + params: { _meta: { [SUBSCRIPTION_ID_META]: l.id } } + })}\n\n` + ); + } + }; + + const server = createServer(async (req, res) => { + if (req.method !== 'POST') { + res.writeHead(405).end(); + return; + } + const body = await readJsonBody(req); + const method = body.method as string; + const id = body.id; + const params = (body.params ?? {}) as Record; + + const send = (result: object) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ + jsonrpc: '2.0', + id, + result: + opts.omitResultType && method.startsWith('events/') + ? result + : withRequiredDraftResultFields(method, result) + }) + ); + }; + const fail = (code: number, message: string, data?: unknown) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ jsonrpc: '2.0', id, error: { code, message, data } }) + ); + }; + + if (method === 'tools/list') { + const obj = { type: 'object', properties: {} }; + const tools = []; + if (opts.durability) { + tools.push( + { name: 'events_conformance_restart', inputSchema: obj }, + { name: 'events_conformance_generation', inputSchema: obj }, + { name: 'events_conformance_subscription_state', inputSchema: obj } + ); + } + if (opts.subscribe?.allowCallbackOriginControl) { + tools.push({ + name: 'events_conformance_allow_callback_origin', + inputSchema: obj + }); + } + if (opts.quota && opts.quota.control !== false) { + tools.push({ name: 'events_conformance_quota', inputSchema: obj }); + } + if (opts.noReplay) { + tools.push({ name: 'events_conformance_yield_gap', inputSchema: obj }); + } + if (opts.listen || opts.catalog) { + tools.push({ name: 'events_conformance_catalog', inputSchema: obj }); + } + if (opts.tenants) { + tools.push( + { name: 'events_conformance_subscribe_as', inputSchema: obj }, + { name: 'events_conformance_subscription_exists', inputSchema: obj } + ); + } + if (opts.webhookEnvelopeControls?.gap) { + tools.push({ + name: 'events_conformance_webhook_gap', + inputSchema: obj + }); + } + if (opts.webhookEnvelopeControls?.terminate) { + tools.push({ + name: 'events_conformance_webhook_terminate', + inputSchema: obj + }); + } + if (tools.length > 0 || opts.durability) { + send({ tools }); + return; + } + } + + if ( + method === 'tools/call' && + (opts.listen || opts.catalog) && + params.name === 'events_conformance_catalog' + ) { + const args = (params.arguments ?? {}) as Record; + const catalog = opts.catalog ?? {}; + /** End every open stream on name with error, as a server terminating them would. */ + const terminateStreams = ( + name: string, + error: { code: number; message: string; data?: unknown } + ) => { + for (const st of [...openStreams]) { + if (st.name !== name) continue; + st.notify('notifications/events/terminated', { error }); + st.stop(); + st.res.end(); + } + }; + if (args.op === 'add') { + catalogMinted += 1; + const name = `catalog.${catalogMinted}`; + names.add(name); + notifyListChanged(); + send({ content: [{ type: 'text', text: `ok: ${name}` }] }); + return; + } + const target = String(args.name ?? ''); + if ( + args.op === 'remove' && + target.startsWith('catalog.') && + names.delete(target) + ) { + if (!catalog.removeSilently) { + terminateStreams(target, { + message: `event type ${target} was removed`, + ...(catalog.removeError ?? { + code: -32023, + data: { kind: 'event' } + }) + }); + } + notifyListChanged(); + send({ content: [{ type: 'text', text: `ok: ${target}` }] }); + return; + } + if ( + (args.op === 'retype' || args.op === 'extend') && + target.startsWith('catalog.') && + names.has(target) + ) { + if (args.op === 'retype' && !catalog.retypeSilently) { + terminateStreams(target, { + message: `event type ${target} changed its inputSchema in place`, + ...(catalog.retypeError ?? { + code: -32026, + data: { feature: 'inputSchema', reason: 'schema_changed' } + }) + }); + } + if (args.op === 'extend' && catalog.extendTerminates) { + terminateStreams(target, { + code: -32026, + message: 'changed', + data: { feature: 'inputSchema', reason: 'schema_changed' } + }); + } + notifyListChanged(); + send({ content: [{ type: 'text', text: `ok: ${target}` }] }); + return; + } + fail(-32602, `events_conformance_catalog cannot ${String(args.op)}`); + return; + } + + if (method === 'subscriptions/listen' && opts.listen) { + const filter = (params.notifications ?? {}) as Record; + const requested = filter.eventsListChanged === true; + const declared = + typeof opts.capability === 'object' && + opts.capability !== null && + (opts.capability as Record).listChanged === true; + const echo = opts.listen.echo ?? 'honor'; + const echoed = + requested && echo !== 'never' && (echo === 'always' || declared); + res.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache' + }); + res.write( + `data: ${JSON.stringify({ + jsonrpc: '2.0', + method: 'notifications/subscriptions/acknowledged', + params: { + _meta: { [SUBSCRIPTION_ID_META]: id }, + notifications: echoed ? { eventsListChanged: true } : {} + } + })}\n\n` + ); + const entry = { res, id, echoed }; + listeners.add(entry); + req.on('close', () => listeners.delete(entry)); + return; + } + + if ( + method === 'tools/call' && + opts.noReplay && + params.name === 'events_conformance_yield_gap' + ) { + const target = String( + ((params.arguments ?? {}) as Record).name ?? '' + ); + const isNoReplay = target === opts.noReplay.name; + for (const s of openStreams) { + if (s.name !== target) continue; + if (!isNoReplay) { + s.notify('notifications/events/active', { + cursor: 'cursor_after_gap', + truncated: true + }); + } else if (opts.noReplay.truncatedOnGap) { + s.notify('notifications/events/active', { + cursor: null, + truncated: true + }); + } + } + send({ content: [{ type: 'text', text: `ok: ${target}` }] }); + return; + } + if ( + method === 'tools/call' && + opts.webhookEnvelopeControls && + (params.name === 'events_conformance_webhook_gap' || + params.name === 'events_conformance_webhook_terminate') + ) { + const isGap = params.name === 'events_conformance_webhook_gap'; + const mode = isGap + ? opts.webhookEnvelopeControls.gap + : opts.webhookEnvelopeControls.terminate; + const toolArgs = (params.arguments ?? {}) as Record; + const entry = [...subscriptions.entries()].find( + ([, sub]) => sub.id === toolArgs.id + ); + const sub = entry?.[1]; + if (!mode || !entry || !sub?.url) { + send({ + content: [{ type: 'text', text: 'no such webhook subscription' }], + isError: true + }); + return; + } + if (!isGap) subscriptions.delete(entry[0]); + if (mode !== 'silent') { + const run = deliverToCallback( + sub.url, + sub.secret, + sub.id, + sub.name ?? '', + { + verify: false, + sendEvent: false, + ...(isGap ? { gapEnvelope: mode } : { terminatedEnvelope: mode }) + } as DeliveryBehaviour + ).finally(() => inFlight.delete(run)); + inFlight.add(run); + } + send({ + content: [{ type: 'text', text: isGap ? 'cursor_after_gap' : 'ok' }] + }); + return; + } + if ( + method === 'tools/call' && + opts.tenants && + (params.name === 'events_conformance_subscribe_as' || + params.name === 'events_conformance_subscription_exists') + ) { + const toolArgs = (params.arguments ?? {}) as Record; + // The run's own principal holds the bare key; another tenant's carries + // its principal in front, unless the key is the defect under test. + const bare = [String(toolArgs.url), String(toolArgs.name), '{}'].join( + '|' + ); + const key = opts.tenants.keyIgnoresPrincipal + ? bare + : `${String(toolArgs.principal)}#${bare}`; + if (params.name === 'events_conformance_subscribe_as') { + const held = subscriptions.get(key); + const subId = held?.id ?? hashKey(key); + subscriptions.set(key, { + id: subId, + url: String(toolArgs.url), + name: String(toolArgs.name) + }); + send({ content: [{ type: 'text', text: subId }] }); + } else { + send({ + content: [{ type: 'text', text: String(subscriptions.has(key)) }] + }); + } + return; + } + if ( + method === 'tools/call' && + params.name === 'events_conformance_quota' && + opts.quota && + opts.quota.control !== false + ) { + const text = JSON.stringify({ + name: opts.quota.name, + max: opts.quota.max + }); + send({ content: [{ type: 'text', text }] }); + return; + } + if ( + method === 'tools/call' && + params.name === 'events_conformance_allow_callback_origin' + ) { + const toolArgs = (params.arguments ?? {}) as Record; + if (typeof toolArgs.origin !== 'string') { + send({ + content: [{ type: 'text', text: 'origin required' }], + isError: true + }); + return; + } + permittedOrigins.add(toolArgs.origin); + send({ content: [{ type: 'text', text: toolArgs.origin }] }); + return; + } + if (opts.durability && method === 'tools/call') { + const d = opts.durability; + const text = (t: string) => + send({ content: [{ type: 'text', text: t }] }); + const tool = params.name; + const toolArgs = (params.arguments ?? {}) as Record; + if (tool === 'events_conformance_restart') { + if (d.restartNoop) { + text(String(generation + 1)); + return; + } + generation++; + for (const [key, sub] of [...subscriptions.entries()]) { + if (sub.noExpiry ? d.dropNoExpiryOnRestart : d.dropFiniteOnRestart) { + subscriptions.delete(key); + } + } + text(String(generation)); + return; + } + if (tool === 'events_conformance_generation') { + text(String(generation)); + return; + } + if (tool === 'events_conformance_subscription_state') { + const entry = [...subscriptions.entries()].find( + ([, sub]) => sub.id === toolArgs.id + ); + if (!entry) { + text('absent'); + return; + } + const [key, sub] = entry; + if ( + d.gcAfterMs !== undefined && + sub.noExpiry && + Date.now() - (sub.at ?? 0) > d.gcAfterMs + ) { + subscriptions.delete(key); + text('absent'); + return; + } + text('active'); + return; + } + fail(-32602, `unknown tool ${String(tool)}`); + return; + } + + if (method === 'server/discover') { + send({ + supportedVersions: [DRAFT_PROTOCOL_VERSION], + capabilities: + 'capability' in opts + ? { extensions: { [EVENTS_EXTENSION_ID]: opts.capability } } + : {}, + serverInfo: { name: 'events-negative', version: '1.0.0' } + }); + return; + } + + if (method === 'events/list') { + if (opts.listError) { + fail(opts.listError.code, opts.listError.message); + return; + } + send({ events: descriptors }); + return; + } + + if (method === 'events/poll') { + polls.push(params); + const name = params.name; + + if (typeof name !== 'string') { + fail(-32602, 'InvalidParams: `name` is required'); + return; + } + if (!names.has(name)) { + fail(opts.unknownNameCode ?? -32023, 'NotFound', { kind: 'event' }); + return; + } + + const byName = opts.pollByName?.[name]; + const chosen = + byName ?? + (queue.length > 1 ? queue.shift()! : (queue[0] ?? pollResult())); + + // Argument validation against the fixture's own declared schema, so the + // invalid-arguments probe has something real to violate. + const args = (params.arguments ?? {}) as Record; + const decl = descriptors.find( + (d) => (d as { name?: unknown }).name === name + ) as { inputSchema?: { properties?: Record } }; + for (const [key, value] of Object.entries(args)) { + const declared = decl?.inputSchema?.properties?.[key]; + if (declared?.type === 'string' && typeof value !== 'string') { + fail(opts.invalidArgsCode ?? -32602, 'InvalidParams'); + return; + } + } + + if ('error' in chosen) { + const e = (chosen as { error: JsonRpcErrorShape }).error; + fail(e.code, e.message); + return; + } + send(chosen as Record); + return; + } + + if (method === 'events/subscribe' || method === 'events/unsubscribe') { + const behaviour = opts.subscribe ?? {}; + const delivery = isRecord(params.delivery) ? params.delivery : {}; + const name = params.name; + const url = delivery.url; + // The document's key, minus the principal a single run cannot vary. + const key = [ + behaviour.idIgnoresUrl ? '' : String(url), + String(name), + JSON.stringify(params.arguments ?? {}) + ].join('|'); + + if (method === 'events/unsubscribe') { + const held = subscriptions.has(key); + if (held && behaviour.unsubscribeHeldCode !== undefined) { + fail(behaviour.unsubscribeHeldCode, 'Unsubscribe refused'); + return; + } + if (held) { + subscriptions.delete(key); + if (opts.tenants?.unsubscribeIgnoresPrincipal) { + for (const k of [...subscriptions.keys()]) { + if (k.endsWith(`#${key}`)) subscriptions.delete(k); + } + } + send({ ...(behaviour.unsubscribeResult ?? {}) }); + return; + } + if (behaviour.unsubscribeUnknownOk) { + send({}); + return; + } + fail(behaviour.unsubscribeUnknownCode ?? -32023, 'NotFound', { + kind: 'subscription' + }); + return; + } + + subscribes.push(params); + if (behaviour.error) { + fail(behaviour.error.code, behaviour.error.message); + return; + } + + const rejectionCode = behaviour.rejectionCode ?? -32602; + const problem = secretProblem(delivery.secret, behaviour); + if (problem) { + fail(rejectionCode, `InvalidParams: \`delivery.secret\` (${problem})`); + return; + } + // A permitted origin bypasses the callback guards, which is what the real + // control does: one origin, both guards, for the rest of the process. + let callbackOrigin = ''; + try { + callbackOrigin = new URL(String(url)).origin; + } catch { + callbackOrigin = ''; + } + const permittedCallback = + callbackOrigin !== '' && permittedOrigins.has(callbackOrigin); + + if ( + !permittedCallback && + !behaviour.acceptHttpUrl && + (typeof url !== 'string' || !url.startsWith('https://')) + ) { + fail(rejectionCode, 'InvalidParams: `delivery.url` must be https'); + return; + } + if ( + !permittedCallback && + behaviour.rejectNonRoutableUrl && + typeof url === 'string' && + nonRoutableHost(url) + ) { + fail( + rejectionCode, + 'InvalidParams: `delivery.url` is not globally routable' + ); + return; + } + + const served = descriptors.find( + (d) => (d as { name?: unknown }).name === name + ) as { delivery?: unknown } | undefined; + if (!served) { + fail(opts.unknownNameCode ?? -32023, 'NotFound', { kind: 'event' }); + return; + } + const modes = Array.isArray(served.delivery) ? served.delivery : []; + if (!modes.includes('webhook') && !behaviour.acceptNonWebhookType) { + fail(-32026, 'Unsupported: event type does not offer webhook delivery'); + return; + } + + const ttl = params.ttlMs; + const reject = behaviour.rejectTtl; + const shortTtl = typeof ttl === 'number' && ttl <= 60_000; + const longTtl = typeof ttl === 'number' && ttl > 7 * 24 * 3600_000; + if ( + (reject === 'both' && (shortTtl || longTtl)) || + (reject === 'short' && shortTtl) || + (reject === 'long' && longTtl) + ) { + fail(-32602, 'InvalidParams: `ttlMs` out of range'); + return; + } + + // An `id` supplied by the caller is a routing handle, never an input, so + // by default it has no bearing on which subscription this addresses. + const addressed = + behaviour.idIsAnInput && typeof params.id === 'string' + ? [...subscriptions.entries()].find( + ([, sub]) => sub.id === params.id + )?.[0] + : undefined; + const effectiveKey = addressed ?? key; + + const existing = subscriptions.get(effectiveKey); + if ( + !existing && + behaviour.maxSubscriptions !== undefined && + subscriptions.size >= behaviour.maxSubscriptions + ) { + fail(-32025, 'ResourceExhausted: subscription cap reached'); + return; + } + const id = + existing && !behaviour.nonIdempotentId + ? existing.id + : behaviour.nonIdempotentId + ? `sub_${++mintedIds}_${hashKey(effectiveKey)}` + : hashKey(effectiveKey); + subscriptions.set(effectiveKey, { + id, + url: typeof url === 'string' ? url : undefined, + secret: delivery.secret, + name: String(name) + }); + + // Clamp rather than reject, and never hand back no-expiry unasked. + const cap = 7 * 24 * 3600_000; + let refreshBefore: unknown; + if (behaviour.refreshBefore !== undefined) { + refreshBefore = behaviour.refreshBefore; + } else if (behaviour.nullRefreshAlways) { + refreshBefore = null; + } else if (ttl === null) { + refreshBefore = null; + } else if (ttl === undefined) { + refreshBefore = behaviour.nullRefreshOnOmitted + ? null + : new Date(Date.now() + 3600_000).toISOString(); + } else { + const granted = behaviour.grantBeyondSuggestion + ? Number(ttl) + 7 * 24 * 3600_000 + : Math.min(Number(ttl), cap); + refreshBefore = new Date(Date.now() + granted).toISOString(); + } + const held = subscriptions.get(effectiveKey); + if (held) { + held.noExpiry = refreshBefore === null; + held.at = held.at ?? Date.now(); + } + + if (opts.delivery?.verifyViaWellKnown && typeof url === 'string') { + await fetchReceiverWellKnown(url); + } else if ( + opts.delivery?.synchronousVerification && + typeof url === 'string' + ) { + const echoed = await challengeCallback(url, delivery.secret, id); + if (!echoed.ok && !opts.delivery.ignoreFailedEcho) { + const leak = opts.delivery.challengeFailureLeaksBody + ? `: endpoint answered ${echoed.body}` + : ''; + fail( + opts.delivery.challengeFailureCode ?? -32027, + `endpoint verification failed${leak}`, + { + reason: opts.delivery.challengeFailureReason ?? 'challenge_failed' + } + ); + return; + } + } + + send({ + ...(behaviour.omitId ? {} : { id }), + ...(behaviour.omitRefreshBefore ? {} : { refreshBefore }), + cursor: 'cursor_sub_001', + truncated: false, + ...deliveryStatusFor(existing !== undefined, url), + ...(behaviour.result ?? {}) + }); + + // Delivery runs after the response, because that is the order a receiver + // sees it in: the subscribe returns, then the callback starts ringing. + if (opts.delivery && typeof url === 'string') { + const run = deliverToCallback( + url, + delivery.secret, + id, + String(name), + opts.delivery.synchronousVerification + ? { ...opts.delivery, verify: false } + : opts.delivery, + (status) => { + if (status >= 200 && status < 300) lastFailureByUrl.delete(url); + else lastFailureByUrl.set(url, status); + } + ).finally(() => inFlight.delete(run)); + inFlight.add(run); + } + return; + } + + if (method === 'events/stream') { + streams.push(params); + const behaviour = { ...CONFORMANT_STREAM, ...(opts.stream ?? {}) }; + const name = params.name; + + if (behaviour.error) { + fail(behaviour.error.code, behaviour.error.message); + return; + } + if ( + !behaviour.acceptAnyName && + (typeof name !== 'string' || !names.has(name)) + ) { + fail(opts.unknownNameCode ?? -32023, 'NotFound', { kind: 'event' }); + return; + } + const servedModes = ( + descriptors.find((d) => (d as { name?: unknown }).name === name) as + | { delivery?: unknown } + | undefined + )?.delivery; + if ( + !behaviour.acceptNonPushType && + Array.isArray(servedModes) && + !servedModes.includes('push') + ) { + fail(-32026, 'Unsupported: event type does not offer push delivery', { + feature: 'deliveryMode', + value: 'push' + }); + return; + } + if (liveStreams >= behaviour.maxConcurrent) { + fail( + -32025, + 'ResourceExhausted: too many subscriptions', + behaviour.capLimitName ? { limit: behaviour.capLimitName } : undefined + ); + return; + } + const quota = opts.quota; + const capped = quota !== undefined && quota.name === name; + if ( + capped && + quota.enforce !== false && + (liveByName.get(quota.name) ?? 0) >= quota.max + ) { + const limit = + quota.limitName === undefined ? 'subscriptions' : quota.limitName; + fail( + -32025, + 'ResourceExhausted: subscription quota reached', + limit === null ? undefined : { limit, max: quota.max } + ); + return; + } + if (behaviour.answerJson) { + send({}); + return; + } + + liveStreams += 1; + const key = String(name); + if (capped) liveByName.set(key, (liveByName.get(key) ?? 0) + 1); + const entry = openStream( + res, + id, + key, + behaviour, + opts.noReplay?.name === key, + opts.omitResultType === true + ); + openStreams.add(entry); + const done = () => { + if (!openStreams.delete(entry)) return; + liveStreams -= 1; + if (capped) liveByName.set(key, (liveByName.get(key) ?? 1) - 1); + entry.stop(); + }; + req.on('close', done); + res.on('close', done); + return; + } + + fail(-32601, `Method not found: ${method}`); + }); + + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, () => resolve()); + }); + const addr = server.address() as AddressInfo; + + return { + url: `http://localhost:${addr.port}/mcp`, + polls, + streams, + subscribes, + liveSubscriptions: () => [...subscriptions.keys()], + async close() { + // Let the callback stop ringing before the receiver goes away, so a + // pending fetch cannot outlive the test that started it. + await Promise.race([ + Promise.allSettled([...inFlight]), + new Promise((resolve) => setTimeout(resolve, 2000)) + ]); + for (const entry of [...openStreams]) { + openStreams.delete(entry); + entry.stop(); + entry.res.destroy(); + } + liveStreams = 0; + for (const l of [...listeners]) { + listeners.delete(l); + l.res.destroy(); + } + server.closeAllConnections?.(); + await new Promise((r) => server.close(() => r())); + } + }; +} + +/** Write the SSE frames a push subscription produces, on timers. */ +function openStream( + res: ServerResponse, + requestId: unknown, + name: string, + behaviour: StreamBehaviour & typeof CONFORMANT_STREAM, + noReplay = false, + omitResultType = false +): { + res: ServerResponse; + stop: () => void; + name: string; + notify: (method: string, params: Record) => void; +} { + const timers: NodeJS.Timeout[] = []; + const stop = () => { + for (const t of timers) clearInterval(t); + timers.length = 0; + }; + + res.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive' + }); + + /** The correlation id goes where the behaviour says, not always in `_meta`. */ + const correlate = (params: Record) => { + if (behaviour.correlation === 'meta') { + return { ...params, _meta: { [SUBSCRIPTION_ID_META]: requestId } }; + } + if (behaviour.correlation === 'requestId') { + return { ...params, requestId }; + } + return params; + }; + const notify = (method: string, params: Record) => { + if (res.writableEnded) return; + res.write( + `data: ${JSON.stringify({ jsonrpc: '2.0', method, params: correlate(params) })}\n\n` + ); + }; + const after = (ms: number, fn: () => void) => { + const t = setTimeout(fn, ms); + timers.push(t); + }; + + if (!behaviour.omitActive) { + notify('notifications/events/active', { + cursor: noReplay ? null : 'cursor_stream_001', + truncated: false, + ...(behaviour.activeParams ?? {}) + }); + } + + if (behaviour.heartbeatMs > 0) { + const beat = setInterval(() => { + notify('notifications/events/heartbeat', { + cursor: 'cursor_stream_001', + ...(behaviour.heartbeatParams ?? {}) + }); + if (behaviour.sseComments && !res.writableEnded) { + res.write(': keepalive\n\n'); + } + }, behaviour.heartbeatMs); + timers.push(beat); + } else if (behaviour.sseComments) { + const beat = setInterval(() => { + if (!res.writableEnded) res.write(': keepalive\n\n'); + }, 150); + timers.push(beat); + } + + if (behaviour.eventAfterMs > 0) { + after(behaviour.eventAfterMs, () => + notify('notifications/events/event', { + ...occurrence({ name }), + ...(behaviour.eventParams ?? {}) + }) + ); + } + + if (behaviour.foreignNotification) { + after(60, () => notify(behaviour.foreignNotification!, {})); + } + if (behaviour.errorNotificationAfterMs) { + after(behaviour.errorNotificationAfterMs, () => + notify('notifications/events/error', { + code: -32603, + message: 'upstream unavailable, retrying' + }) + ); + } + if (behaviour.terminatedAfterMs) { + after(behaviour.terminatedAfterMs, () => + notify('notifications/events/terminated', { reason: 'revoked' }) + ); + } + if (behaviour.gapAfterMs) { + after(behaviour.gapAfterMs, () => + notify('notifications/events/active', { + cursor: 'cursor_stream_002', + truncated: true + }) + ); + } + if (behaviour.closeAfterMs) { + after(behaviour.closeAfterMs, () => { + if (res.writableEnded) return; + res.write( + `data: ${JSON.stringify({ + jsonrpc: '2.0', + id: requestId, + result: behaviour.finalResult ?? { + ...(omitResultType ? {} : { resultType: 'complete' }), + _meta: { [SUBSCRIPTION_ID_META]: requestId } + } + })}\n\n` + ); + stop(); + res.end(); + }); + } + + return { res, stop, name, notify }; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** Retry spacing. Over a second, because `webhook-timestamp` is in seconds and + * two attempts inside one second would share a stamp the fixture did freshen. */ +const RETRY_GAP_MS = 1100; + +/** Spacing between follow-up events, so they read as a cadence rather than a burst. */ +const FOLLOW_UP_GAP_MS = 300; + +/** + * The handshake as a server that verifies inside `events/subscribe` runs it: + * one signed `verification` POST, and true only for a 2xx whose body echoes the + * nonce. No retries and no redirects, the same as a delivery. + */ +async function fetchReceiverWellKnown(callbackUrl: string): Promise { + try { + const origin = new URL(callbackUrl).origin; + const res = await fetch(`${origin}/.well-known/mcp-webhook-receiver.json`); + if (!res.ok) { + await res.text(); + return false; + } + const doc: unknown = await res.json(); + const prefixes = + isRecord(doc) && Array.isArray(doc.receivers) ? doc.receivers : []; + const path = new URL(callbackUrl).pathname; + return prefixes.some((p) => typeof p === 'string' && path.startsWith(p)); + } catch { + return false; + } +} + +async function challengeCallback( + url: string, + secret: unknown, + subscriptionId: string +): Promise<{ ok: boolean; body: string }> { + const key = + typeof secret === 'string' && secret.startsWith('whsec_') + ? Buffer.from(secret.slice('whsec_'.length), 'base64') + : Buffer.from(String(secret ?? '')); + const challenge = `chal_${Math.random().toString(36).slice(2, 14)}`; + const body = JSON.stringify({ type: 'verification', challenge }); + const webhookId = `msg_verification_${Math.random().toString(36).slice(2, 10)}`; + const stamp = String(Math.floor(Date.now() / 1000)); + const signature = `v1,${createHmac('sha256', key).update(`${webhookId}.${stamp}.${body}`).digest('base64')}`; + try { + const res = await fetch(url, { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'webhook-id': webhookId, + 'webhook-timestamp': stamp, + 'webhook-signature': signature, + 'x-mcp-subscription-id': subscriptionId + }, + body, + redirect: 'manual' + }); + const text = await res.text(); + if (res.status < 200 || res.status >= 300) return { ok: false, body: text }; + const reply: unknown = JSON.parse(text); + return { + ok: isRecord(reply) && reply.challenge === challenge, + body: text + }; + } catch (err) { + return { ok: false, body: String(err) }; + } +} + +/** + * POST the verification challenge and then the event, the way the document says + * to, with whatever this behaviour breaks. + * + * Errors are swallowed: the callback is a receiver the scenario owns, it may + * refuse or vanish mid-run by design, and a fixture that threw here would fail + * the test rather than the check under test. + */ +async function deliverToCallback( + url: string, + secret: unknown, + subscriptionId: string, + eventName: string, + behaviour: DeliveryBehaviour, + onStatus?: (status: number) => void +): Promise { + let gone = false; + const key = + typeof secret === 'string' && secret.startsWith('whsec_') + ? behaviour.literalKeySignature + ? Buffer.from(secret) + : Buffer.from(secret.slice('whsec_'.length), 'base64') + : Buffer.from(String(secret ?? '')); + + const headersFor = (webhookId: string, timestamp: string, body: string) => { + const omit = new Set(behaviour.omitHeaders ?? []); + const signature = `v1,${createHmac('sha256', key).update(`${webhookId}.${timestamp}.${body}`).digest('base64')}`; + const headers: Record = { + 'content-type': behaviour.contentType ?? 'application/json' + }; + if (!omit.has('webhook-id')) headers['webhook-id'] = webhookId; + if (!omit.has('webhook-timestamp')) + headers['webhook-timestamp'] = timestamp; + if (!omit.has('webhook-signature')) + headers['webhook-signature'] = signature; + if (!behaviour.omitSubscriptionIdHeader) { + headers['x-mcp-subscription-id'] = behaviour.wrongSubscriptionIdHeader + ? 'sub_someone_elses' + : subscriptionId; + } + return headers; + }; + + /** One delivery, with the retry rules applied to whatever it answers. */ + const post = async ( + webhookId: string, + body: string, + opts: { retryable?: boolean } = {} + ): Promise => { + const attempts = Math.max(1, behaviour.attempts ?? 3); + let stamp = String(Math.floor(Date.now() / 1000)); + let target = url; + + for (let attempt = 1; attempt <= attempts; attempt++) { + if (attempt > 1 && !behaviour.staleRetrySignature) { + stamp = String(Math.floor(Date.now() / 1000)); + } + let status: number; + let location: string | null = null; + try { + const res = await fetch(target, { + method: behaviour.method ?? 'POST', + headers: headersFor(webhookId, stamp, body), + body, + redirect: 'manual' + }); + status = res.status; + location = res.headers.get('location'); + await res.text(); + onStatus?.(status); + if (status === 410) gone = true; + } catch { + return; + } + + if (status === 302 && location) { + // A conformant server stops here. Following is the SSRF hazard the + // no-redirects rule exists for. + if (!behaviour.followRedirects) return; + target = location; + continue; + } + if (status >= 200 && status < 300) return; + if ((status === 410 || status === 413) && !behaviour.retryNonRetryable) { + return; + } + if (opts.retryable === false) return; + if (attempt === attempts) return; + await new Promise((resolve) => + setTimeout(resolve, behaviour.retryGapMs ?? RETRY_GAP_MS) + ); + } + }; + + const envelopeId = (type: string) => + behaviour.envelopeIdFormat === 'plain' + ? `wh-${Math.random().toString(36).slice(2, 10)}` + : `msg_${type}_${Math.random().toString(36).slice(2, 10)}`; + + /** Control envelopes are signed and headed exactly like deliveries — unless + * this behaviour is the one that says otherwise. */ + const postEnvelope = async ( + type: string, + body: Record + ): Promise => { + const json = JSON.stringify({ type, ...body }); + if (behaviour.signEnvelopes === false) { + try { + await fetch(url, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: json, + redirect: 'manual' + }).then((r) => r.text()); + } catch { + // See above: a receiver refusing is the scenario's business. + } + return; + } + await post(envelopeId(type), json); + }; + + const postEvent = async (): Promise => { + const data = behaviour.oversizedBody + ? { id: 'x', padding: 'p'.repeat(300 * 1024) } + : { id: 'x' }; + await post( + envelopeId('event'), + JSON.stringify({ + eventId: `evt_${Math.random().toString(36).slice(2, 10)}`, + name: eventName, + timestamp: new Date().toISOString(), + data + }) + ); + }; + + if (behaviour.eventBeforeVerification) { + await postEvent(); + // The check compares millisecond arrival times, and two loopback POSTs land + // inside the same millisecond often enough to make the out-of-order case + // read as in-order. A real server delivering before it verifies is not this + // close, so the gap is realism rather than a thumb on the scale. + await new Promise((resolve) => setTimeout(resolve, 60)); + } + if (behaviour.verify !== false) { + await postEnvelope('verification', { + challenge: `chal_${Math.random().toString(36).slice(2, 14)}`, + subscriptionId + }); + } + if (behaviour.sendEvent !== false && !behaviour.eventBeforeVerification) { + await postEvent(); + for (let i = 0; i < (behaviour.followUpEvents ?? 0); i++) { + await new Promise((resolve) => setTimeout(resolve, FOLLOW_UP_GAP_MS)); + if (behaviour.stopAfterGone && gone) break; + await postEvent(); + } + } + if (behaviour.gapEnvelope) { + const override = isRecord(behaviour.gapEnvelope) + ? behaviour.gapEnvelope + : {}; + await postEnvelope('gap', { + cursor: 'cursor_after_gap', + ...override + }); + } + if (behaviour.terminatedEnvelope) { + const override = isRecord(behaviour.terminatedEnvelope) + ? behaviour.terminatedEnvelope + : {}; + await postEnvelope('terminated', { + error: { code: -32024, message: 'authorization revoked' }, + ...override + }); + } +} + +/** A deterministic id over the subscription key, which is what the document asks for. */ +function hashKey(key: string): string { + return `sub_${createHash('sha256').update(key).digest('hex').slice(0, 24)}`; +} + +export async function readJsonBody( + req: IncomingMessage +): Promise> { + const chunks: Buffer[] = []; + for await (const chunk of req) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record< + string, + unknown + >; +} + +/** Literal hosts a hardened server refuses without resolving anything. */ +function nonRoutableHost(url: string): boolean { + let host: string; + try { + host = new URL(url).hostname.replace(/^\[|\]$/g, '').toLowerCase(); + } catch { + return false; + } + if (host === 'localhost' || host === '::1' || host === '::') return true; + const v4 = host.match(/^(\d+)\.(\d+)\.\d+\.\d+$/); + if (!v4) return /^f[cd]|^fe80:/.test(host); + const [a, b] = [Number(v4[1]), Number(v4[2])]; + return ( + a === 127 || + a === 10 || + a === 0 || + (a === 169 && b === 254) || + (a === 172 && b >= 16 && b <= 31) || + (a === 192 && b === 168) + ); +} diff --git a/src/scenarios/server/events/negative-push.test.ts b/src/scenarios/server/events/negative-push.test.ts new file mode 100644 index 00000000..88f9f7af --- /dev/null +++ b/src/scenarios/server/events/negative-push.test.ts @@ -0,0 +1,668 @@ +import { describe, test, expect, vi, beforeAll, afterAll } from 'vitest'; +import { DRAFT_PROTOCOL_VERSION } from '../../../types'; +import { + descriptor, + startEventsFixture, + type CatalogBehaviour, + type EventsFixtureOptions, + type StreamBehaviour +} from './negative-fixture'; + +/** + * Negative controls for `events-push`. + * + * A passing run against kitchen-sink proves the scenario does not + * false-positive. It does not prove any check catches anything, which is what + * these are for: each case pairs the conformant fixture with one broken in + * exactly one way and asserts the check flips. + * + * Four rows report untestable against both real implementations, because no + * client can ask a server to fail upstream, lose its replay window, terminate a + * subscription or close a stream. The fixture can do all four on demand, so + * they are graded here rather than only declared — which is the only evidence + * that those checks work at all. + * + * `EVENTS_PUSH_WATCH_MS` is stubbed down from 35s, since the scenario reads it + * when the module is first evaluated, and `vi.resetModules()` before the dynamic + * import is what makes the stub land regardless of whether another test file + * imported the scenario first. Two catches follow from that. A reset registry + * hands back a second copy of the connection module, and `err instanceof + * JsonRpcError` is false across two copies of the same class, so the run context + * has to come from the same fresh graph as the scenario rather than a static + * import up here. And the window is fixed at import, so one graph means one + * window. + * + * The fast cases share one 900ms graph, imported once, and run concurrently: + * each builds its own fixture on its own port and touches nothing shared. The + * single case that has to outlast the 30s cadence the document permits takes its + * own graph, sequentially, at the foot of the file — a `resetModules()` while + * the concurrent cases were still running would pull their graph out from under + * them. + */ + +/** The window every case uses, except the one that needs to outlast 30s. */ +const FAST_WATCH_MS = 900; + +type Imported = { + Scenario: typeof import('./push').EventsPushScenario; + testContext: typeof import('../../../connection/testing').testContext; + takeWireViolations: typeof import('../../../validation/wire-schema').takeWireViolations; +}; + +/** The scenario, with one watch window, in its own module registry. */ +async function importWith(watchMs: number): Promise { + vi.stubEnv('EVENTS_PUSH_WATCH_MS', String(watchMs)); + vi.resetModules(); + const [scenario, testing, wire] = await Promise.all([ + import('./push'), + import('../../../connection/testing'), + import('../../../validation/wire-schema') + ]); + return { + Scenario: scenario.EventsPushScenario, + testContext: testing.testContext, + takeWireViolations: wire.takeWireViolations + }; +} + +let fast: Imported; + +beforeAll(async () => { + fast = await importWith(FAST_WATCH_MS); +}); + +afterAll(() => { + vi.unstubAllEnvs(); +}); + +async function pushChecks(opts: EventsFixtureOptions, using: Imported = fast) { + const fixture = await startEventsFixture(opts); + try { + const checks = await new using.Scenario().run( + using.testContext(fixture.url, DRAFT_PROTOCOL_VERSION) + ); + // Drained so an intentionally malformed frame does not trip the global + // vitest hook; these tests assert on the check, not the wire validator. + using.takeWireViolations(); + return new Map(checks.map((c) => [c.id, c])); + } finally { + await fixture.close(); + } +} + +/** A push-capable catalog, which is the only thing this scenario selects on. */ +function pushFixture(stream: StreamBehaviour = {}): EventsFixtureOptions { + return { + capability: { listChanged: true }, + descriptors: [descriptor({ name: 'push.event', delivery: ['push'] })], + stream + }; +} + +describe.concurrent('events/stream opening', () => { + test('the conformant fixture passes every gradeable row', async () => { + const checks = await pushChecks(pushFixture()); + for (const id of [ + 'sep-3415-stream-implemented', + 'sep-3415-stream-active-confirmation', + 'sep-3415-stream-subscription-id-meta', + 'sep-3415-stream-event-notification', + 'sep-3415-stream-carries-only-event-notifications', + 'sep-3415-stream-heartbeat-required', + 'sep-3415-stream-heartbeat-carries-cursor', + 'sep-3415-stream-heartbeat-interval', + 'sep-3415-stream-heartbeat-not-sse-comment', + 'sep-3415-stream-cancel-stops-delivery', + 'sep-3415-stream-exempt-from-concurrency-cap', + 'sep-3415-stream-error-before-open' + ]) { + expect(checks.get(id)?.status, id).toBe('SUCCESS'); + } + }); + + test('a server with no push-capable type reports every row untestable', async () => { + const checks = await pushChecks({ + capability: { listChanged: true }, + descriptors: [descriptor()] + }); + const check = checks.get('sep-3415-stream-implemented'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('No event type advertises'); + expect(check?.details?.untestable).toBe(true); + }); + + const ROW = 'sep-3415-stream-mode-unsupported'; + const withPollOnly = ( + stream: StreamBehaviour = {} + ): EventsFixtureOptions => ({ + ...pushFixture(stream), + descriptors: [ + descriptor({ name: 'push.event', delivery: ['push'] }), + descriptor({ name: 'poll.only', delivery: ['poll'] }) + ] + }); + + test('a type that declines push is refused with -32026', async () => { + const checks = await pushChecks(withPollOnly()); + expect(checks.get(ROW)?.status, checks.get(ROW)?.errorMessage).toBe( + 'SUCCESS' + ); + }); + + // The defect in #1417: the stream handler gated on whether the source could + // stream, not on what its descriptor declared. + test('streaming a type that declines push fails the row', async () => { + const checks = await pushChecks(withPollOnly({ acceptNonPushType: true })); + expect(checks.get(ROW)?.status).toBe('FAILURE'); + expect(checks.get(ROW)?.errorMessage).toContain('opened a stream'); + }); + + test('a poll-only server still grades the unsupported-mode row', async () => { + const checks = await pushChecks({ + capability: { listChanged: true }, + descriptors: [descriptor({ name: 'poll.only', delivery: ['poll'] })] + }); + expect(checks.get(ROW)?.status).toBe('SUCCESS'); + }); + + test('with every type offering push, the row is untestable', async () => { + const checks = await pushChecks(pushFixture()); + expect(checks.get(ROW)?.details?.untestable).toBe(true); + }); + + test('a server that declares nothing and serves nothing skips the suite', async () => { + const checks = await pushChecks({ + listError: { code: -32601, message: 'Method not found' } + }); + for (const check of checks.values()) { + expect(check.status).toBe('SKIPPED'); + } + }); + + test('answering events/stream with a JSON result rather than a stream fails', async () => { + const checks = await pushChecks(pushFixture({ answerJson: true })); + const check = checks.get('sep-3415-stream-implemented'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('expected an SSE stream'); + // And nothing on the stream is reported green just because it was silent. + expect(checks.get('sep-3415-stream-heartbeat-required')?.status).toBe( + 'FAILURE' + ); + expect( + checks.get('sep-3415-stream-heartbeat-required')?.details?.untestable + ).toBe(true); + }); + + test('refusing a valid subscription outright fails and names the code', async () => { + const checks = await pushChecks( + pushFixture({ error: { code: -32025, message: 'ResourceExhausted' } }) + ); + const check = checks.get('sep-3415-stream-implemented'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('-32025'); + }); + + test('opening a stream for an unknown event type fails error-before-open', async () => { + const checks = await pushChecks(pushFixture({ acceptAnyName: true })); + const check = checks.get('sep-3415-stream-error-before-open'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('did not answer a JSON-RPC error'); + }); + + test('answering an unknown type with -32026 rather than NotFound warns', async () => { + const checks = await pushChecks({ + ...pushFixture(), + unknownNameCode: -32026 + }); + const check = checks.get('sep-3415-stream-error-before-open'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('-32023 NotFound'); + }); +}); + +describe.concurrent('the active confirmation', () => { + test('no confirmation at all fails', async () => { + const checks = await pushChecks(pushFixture({ omitActive: true })); + const check = checks.get('sep-3415-stream-active-confirmation'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('notifications/events/active'); + }); + + test('a numeric cursor on the confirmation fails', async () => { + const checks = await pushChecks( + pushFixture({ activeParams: { cursor: 42 } }) + ); + const check = checks.get('sep-3415-stream-active-confirmation'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('`cursor` is'); + }); + + test('a string truncated flag fails', async () => { + const checks = await pushChecks( + pushFixture({ activeParams: { truncated: 'no' } }) + ); + const check = checks.get('sep-3415-stream-active-confirmation'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('`truncated` is'); + }); +}); + +describe.concurrent('the correlation id', () => { + // The divergence this row exists for: mcpkit puts the id in params.requestId, + // mirroring the sketch's own push examples, where the document requires the + // SEP-2575 `_meta` spelling. A client holding two streams cannot route by + // what the document tells it to read. + test('params.requestId instead of _meta fails and names the key', async () => { + const checks = await pushChecks(pushFixture({ correlation: 'requestId' })); + const check = checks.get('sep-3415-stream-subscription-id-meta'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain( + 'io.modelcontextprotocol/subscriptionId' + ); + }); + + test('no correlation id at all fails', async () => { + const checks = await pushChecks(pushFixture({ correlation: 'none' })); + expect(checks.get('sep-3415-stream-subscription-id-meta')?.status).toBe( + 'FAILURE' + ); + }); +}); + +describe.concurrent('what rides the stream', () => { + test('a non-events notification fails and names the method', async () => { + const checks = await pushChecks( + pushFixture({ foreignNotification: 'notifications/message' }) + ); + const check = checks.get( + 'sep-3415-stream-carries-only-event-notifications' + ); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('notifications/message'); + }); + + test('a malformed occurrence fails the event row', async () => { + const checks = await pushChecks( + pushFixture({ eventParams: { timestamp: 'last tuesday' } }) + ); + const check = checks.get('sep-3415-stream-event-notification'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('EventOccurrence'); + }); + + test('a stream that delivers nothing reports the event row untestable', async () => { + const checks = await pushChecks(pushFixture({ eventAfterMs: 0 })); + const check = checks.get('sep-3415-stream-event-notification'); + expect(check?.status).toBe('FAILURE'); + expect(check?.details?.untestable).toBe(true); + }); +}); + +describe.concurrent('the heartbeat', () => { + test('a numeric cursor on the heartbeat fails', async () => { + const checks = await pushChecks( + pushFixture({ heartbeatParams: { cursor: 42 } }) + ); + const check = checks.get('sep-3415-stream-heartbeat-carries-cursor'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('neither a string nor null'); + }); + + test('silence inside a short window is untestable, not a failure', async () => { + const checks = await pushChecks(pushFixture({ heartbeatMs: 0 })); + const check = checks.get('sep-3415-stream-heartbeat-required'); + expect(check?.status).toBe('FAILURE'); + expect(check?.details?.untestable).toBe(true); + expect(check?.errorMessage).toContain('EVENTS_PUSH_WATCH_MS'); + }); + + test('an SSE comment keepalive beside a data heartbeat warns', async () => { + const checks = await pushChecks(pushFixture({ sseComments: true })); + const check = checks.get('sep-3415-stream-heartbeat-not-sse-comment'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('keepalive'); + }); + + test('an SSE comment instead of a data heartbeat fails', async () => { + const checks = await pushChecks( + pushFixture({ heartbeatMs: 0, sseComments: true }) + ); + expect( + checks.get('sep-3415-stream-heartbeat-not-sse-comment')?.status + ).toBe('FAILURE'); + }); +}); + +describe.concurrent('the -32025 error-table row', () => { + test('a cap that names its limit passes, and one that does not warns', async () => { + const named = await pushChecks( + pushFixture({ maxConcurrent: 1, capLimitName: 'subscriptions' }) + ); + const ok = named.get('sep-3415-error-resource-exhausted'); + expect(ok?.status).toBe('SUCCESS'); + expect(ok?.details?.limit).toBe('subscriptions'); + + const vague = await pushChecks(pushFixture({ maxConcurrent: 1 })); + const check = vague.get('sep-3415-error-resource-exhausted'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('which quota it hit'); + }); + + test('a server that refuses nothing reports the row untestable', async () => { + const checks = await pushChecks(pushFixture()); + const check = checks.get('sep-3415-error-resource-exhausted'); + expect(check?.details?.untestable).toBe(true); + expect(check?.errorMessage).toContain('never provoked'); + }); +}); + +describe.concurrent( + 'the -32025 row, probed on the type a server says is capped', + () => { + // Two push types: the scenario's target, which the concurrency probe opens + // three streams on, and a capped one it must leave alone. + const withQuota = ( + quota: EventsFixtureOptions['quota'] + ): EventsFixtureOptions => ({ + capability: { listChanged: true }, + descriptors: [ + descriptor({ name: 'push.event', delivery: ['push'] }), + descriptor({ name: 'capped.event', delivery: ['push'] }) + ], + quota + }); + + test('an enforced cap that names its limit passes, beside a passing concurrency row', async () => { + const checks = await pushChecks( + withQuota({ name: 'capped.event', max: 2 }) + ); + const check = checks.get('sep-3415-error-resource-exhausted'); + expect(check?.status).toBe('SUCCESS'); + expect(check?.details?.limit).toBe('subscriptions'); + expect( + checks.get('sep-3415-stream-exempt-from-concurrency-cap')?.status + ).toBe('SUCCESS'); + }); + + test('a refusal without data.limit warns', async () => { + const checks = await pushChecks( + withQuota({ name: 'capped.event', max: 1, limitName: null }) + ); + const check = checks.get('sep-3415-error-resource-exhausted'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('which quota it hit'); + }); + + test('a reported cap that is never enforced is untestable, and says so', async () => { + const checks = await pushChecks( + withQuota({ name: 'capped.event', max: 2, enforce: false }) + ); + const check = checks.get('sep-3415-error-resource-exhausted'); + expect(check?.status).toBe('FAILURE'); + expect(check?.details?.untestable).toBe(true); + expect(check?.errorMessage).toContain('3 streams opened'); + }); + + test('without the control the row stays untestable and names it', async () => { + const checks = await pushChecks( + withQuota({ name: 'capped.event', max: 2, control: false }) + ); + const check = checks.get('sep-3415-error-resource-exhausted'); + expect(check?.details?.untestable).toBe(true); + expect(check?.errorMessage).toContain('events_conformance_quota'); + }); + } +); + +describe.concurrent('truncated on a type without replay', () => { + const withNoReplay = ( + noReplay: EventsFixtureOptions['noReplay'] + ): EventsFixtureOptions => ({ + capability: { listChanged: true }, + descriptors: [ + descriptor({ name: 'push.event', delivery: ['push'] }), + descriptor({ name: 'quiet.event', delivery: ['push'] }) + ], + noReplay + }); + const id = 'sep-3415-truncated-false-when-no-replay'; + + test('a gap that sends nothing on a null-cursor stream passes', async () => { + const checks = await pushChecks(withNoReplay({ name: 'quiet.event' })); + const check = checks.get(id); + expect(check?.status).toBe('SUCCESS'); + expect(check?.details?.name).toBe('quiet.event'); + }); + + test('a gap that sends truncated:true on a null-cursor stream warns', async () => { + const checks = await pushChecks( + withNoReplay({ name: 'quiet.event', truncatedOnGap: true }) + ); + const check = checks.get(id); + expect(check?.status).toBe('WARNING'); + expect(check?.details?.untestable).toBeUndefined(); + expect(check?.details?.frame).toMatchObject({ + cursor: null, + truncated: true + }); + }); + + test('a catalog where every type replays makes the rule not apply', async () => { + const checks = await pushChecks(withNoReplay({ name: 'absent.event' })); + expect(checks.get(id)?.status).toBe('SKIPPED'); + }); + + test('without the gap control the row is untestable and names it', async () => { + const checks = await pushChecks(pushFixture()); + const check = checks.get(id); + expect(check?.details?.untestable).toBe(true); + expect(check?.errorMessage).toContain('events_conformance_yield_gap'); + }); + + test('an early return still reports the row', async () => { + const checks = await pushChecks({ + capability: { listChanged: true }, + listError: { code: -32603, message: 'boom' } + }); + expect(checks.get(id)?.details?.untestable).toBe(true); + }); +}); + +describe.concurrent('concurrency and cancellation', () => { + // The cap kitchen-sink applies to streams, which the document exempts them + // from: the first stream confirms and the other two are refused -32025. + test('a per-principal cap that catches streams fails', async () => { + const checks = await pushChecks(pushFixture({ maxConcurrent: 1 })); + const check = checks.get('sep-3415-stream-exempt-from-concurrency-cap'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('only 1 confirmed'); + }); + + test('a server that keeps sending after the abort fails cancel-stops-delivery', async () => { + // Nothing in the fixture can outlive an aborted request, so the row is + // proven the other way round: it passes here, and the failure branch is + // reachable only by a server that holds the connection open after abort. + const checks = await pushChecks(pushFixture()); + expect(checks.get('sep-3415-stream-cancel-stops-delivery')?.status).toBe( + 'SUCCESS' + ); + }); +}); + +describe.concurrent('rows that are untestable against a real server', () => { + test('a server-initiated close grades the final result instead of skipping it', async () => { + const cancelled = await pushChecks(pushFixture()); + const untested = cancelled.get('sep-3415-stream-final-result-shape'); + expect(untested?.status).toBe('WARNING'); + expect(untested?.details?.untestable).toBe(true); + + const closed = await pushChecks(pushFixture({ closeAfterMs: 400 })); + expect(closed.get('sep-3415-stream-final-result-shape')?.status).toBe( + 'SUCCESS' + ); + expect(closed.get('sep-3415-stream-final-result-timing')?.status).toBe( + 'SUCCESS' + ); + }); + + test('a final result carrying fields fails, since it is defined as empty', async () => { + const checks = await pushChecks( + pushFixture({ closeAfterMs: 400, finalResult: { events: [] } }) + ); + const check = checks.get('sep-3415-stream-final-result-shape'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('events'); + }); + + test('a final result without the subscription ID in _meta fails only the shape row', async () => { + const ok = await pushChecks(pushFixture({ closeAfterMs: 400 })); + expect(ok.get('sep-3415-stream-final-result-shape')?.status).toBe( + 'SUCCESS' + ); + expect(ok.get('sep-3415-result-type-complete')?.status).toBe('SUCCESS'); + + const bare = await pushChecks( + pushFixture({ + closeAfterMs: 400, + finalResult: { resultType: 'complete', _meta: {} } + }) + ); + const check = bare.get('sep-3415-stream-final-result-shape'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain( + 'io.modelcontextprotocol/subscriptionId' + ); + expect(bare.get('sep-3415-result-type-complete')?.status).toBe('SUCCESS'); + }); + + test('a final result without resultType fails only the resultType row', async () => { + const checks = await pushChecks({ + ...pushFixture({ closeAfterMs: 400 }), + omitResultType: true + }); + const check = checks.get('sep-3415-result-type-complete'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('events/stream'); + expect(checks.get('sep-3415-stream-final-result-shape')?.status).toBe( + 'SUCCESS' + ); + }); + + test('an error notification, a termination and a gap are all graded when they happen', async () => { + const quiet = await pushChecks(pushFixture()); + for (const id of [ + 'sep-3415-stream-error-is-recoverable', + 'sep-3415-stream-terminated-ends-subscription', + 'sep-3415-stream-gap-resends-active' + ]) { + expect(quiet.get(id)?.details?.untestable, id).toBe(true); + } + + const busy = await pushChecks( + pushFixture({ + errorNotificationAfterMs: 150, + terminatedAfterMs: 250, + gapAfterMs: 350 + }) + ); + for (const id of [ + 'sep-3415-stream-error-is-recoverable', + 'sep-3415-stream-terminated-ends-subscription', + 'sep-3415-stream-gap-resends-active' + ]) { + expect(busy.get(id)?.status, id).toBe('SUCCESS'); + } + }); +}); + +/** + * Under 30s a silent server and a slow one are indistinguishable, so this is the + * only window in which the heartbeat MUST can fail rather than report a window + * too short to judge. It costs half a minute, which is why it is one case and + * not a pair, and why it runs last on a graph of its own. + */ +describe('the heartbeat MUST, on a window that outlasts the cadence', () => { + test('silence past 30s fails instead of reporting untestable', async () => { + const slow = await importWith(30_050); + const checks = await pushChecks(pushFixture({ heartbeatMs: 0 }), slow); + const check = checks.get('sep-3415-stream-heartbeat-required'); + expect(check?.status).toBe('FAILURE'); + expect(check?.details?.untestable).toBeUndefined(); + expect(check?.errorMessage).toContain('outlasts the 30s cadence'); + }, 90_000); +}); + +describe.concurrent('a catalog that changes under a live stream', () => { + const ROWS = { + terminates: 'sep-3415-removal-terminates-subscriptions', + notFound: 'sep-3415-removal-error-not-found', + schemaChanged: 'sep-3415-removal-error-schema-changed', + additive: 'sep-3415-removal-additive-no-terminate' + }; + const all = Object.values(ROWS); + const withCatalog = ( + catalog: CatalogBehaviour = {} + ): EventsFixtureOptions => ({ + ...pushFixture(), + catalog + }); + /** Every catalog row except `except` is SUCCESS. */ + const othersPass = ( + checks: Map, + except: string[] + ) => { + for (const id of all.filter((r) => !except.includes(r))) { + expect(checks.get(id)?.status, id).toBe('SUCCESS'); + } + }; + + test('the conformant catalog passes all four rows', async () => { + const checks = await pushChecks(withCatalog()); + othersPass(checks, []); + }); + + test('without the control, all four are untestable', async () => { + const checks = await pushChecks(pushFixture()); + for (const id of all) { + expect(checks.get(id)?.details?.untestable, id).toBe(true); + } + }); + + test('a removal that ends no stream warns, and leaves NotFound ungraded', async () => { + const checks = await pushChecks(withCatalog({ removeSilently: true })); + expect(checks.get(ROWS.terminates)?.status).toBe('WARNING'); + expect(checks.get(ROWS.notFound)?.details?.untestable).toBe(true); + othersPass(checks, [ROWS.terminates, ROWS.notFound]); + }); + + test('ending a removed type with Forbidden fails NotFound', async () => { + const checks = await pushChecks( + withCatalog({ removeError: { code: -32024 } }) + ); + expect(checks.get(ROWS.notFound)?.status).toBe('FAILURE'); + othersPass(checks, [ROWS.notFound]); + }); + + test('ending a retyped type with Forbidden fails schema_changed', async () => { + const checks = await pushChecks( + withCatalog({ retypeError: { code: -32024 } }) + ); + expect(checks.get(ROWS.schemaChanged)?.status).toBe('FAILURE'); + expect(checks.get(ROWS.schemaChanged)?.errorMessage).toContain( + 'Not Forbidden' + ); + othersPass(checks, [ROWS.schemaChanged]); + }); + + test('a retype that ends nothing leaves schema_changed untestable', async () => { + const checks = await pushChecks(withCatalog({ retypeSilently: true })); + expect(checks.get(ROWS.schemaChanged)?.details?.untestable).toBe(true); + othersPass(checks, [ROWS.schemaChanged]); + }); + + test('an additive change that ends the stream fails', async () => { + const checks = await pushChecks(withCatalog({ extendTerminates: true })); + expect(checks.get(ROWS.additive)?.status).toBe('FAILURE'); + othersPass(checks, [ROWS.additive]); + }); +}); diff --git a/src/scenarios/server/events/negative-webhook.test.ts b/src/scenarios/server/events/negative-webhook.test.ts new file mode 100644 index 00000000..782863c1 --- /dev/null +++ b/src/scenarios/server/events/negative-webhook.test.ts @@ -0,0 +1,601 @@ +import { describe, test, expect, beforeAll, afterAll, vi } from 'vitest'; +import { testContext } from '../../../connection/testing'; +import { DRAFT_PROTOCOL_VERSION } from '../../../types'; +import { takeWireViolations } from '../../../validation/wire-schema'; +import { EventsWebhookScenario } from './webhook'; +import { + descriptor, + startEventsFixture, + type EventsFixtureOptions, + type SubscribeBehaviour +} from './negative-fixture'; + +/** + * Negative controls for `events-webhook`. + * + * Each case pairs the conformant fixture with one broken in exactly one way. + * Where a single defect costs more than one row, the test says so rather than + * pretending the rows are independent: a server that returns no `id` at all + * takes the key-composition rows down with it, because there is nothing left to + * compare. + * + * Two of these are divergences the suite found in a real implementation, and + * both are here so a red run can be trusted: an `http://` callback accepted at + * subscribe, and an unsubscribe of a key the server never held answering + * success. + */ + +const ALL_ROWS = 30; + +// The durability rows poll for a restart and for a GC drop; the defaults are +// sized for a real server and would make this file crawl. +beforeAll(() => { + vi.stubEnv('EVENTS_RESTART_WAIT_MS', '1500'); + vi.stubEnv('EVENTS_GC_WAIT_MS', '2000'); +}); +afterAll(() => { + vi.unstubAllEnvs(); +}); + +/** Rows every conformant run passes, which is the whole surface bar the seven + * that need a second principal or a restart. */ +const GRADEABLE = [ + 'sep-3415-result-type-complete', + 'sep-3415-subscribe-webhook-only', + 'sep-3415-subscribe-secret-required', + 'sep-3415-subscribe-secret-format', + 'sep-3415-subscribe-secret-rejected', + 'sep-3415-subscribe-url-https-required', + 'sep-3415-subscribe-url-non-https-rejected', + 'sep-3415-subscribe-key-composition', + 'sep-3415-subscribe-key-immutable', + 'sep-3415-subscribe-idempotent-upsert', + 'sep-3415-subscribe-id-derived', + 'sep-3415-subscribe-id-not-an-input', + 'sep-3415-subscribe-response-cursor', + 'sep-3415-subscribe-response-truncated', + 'sep-3415-subscribe-refresh-before-present', + 'sep-3415-ttl-omitted-means-default', + 'sep-3415-ttl-null-only-when-requested', + 'sep-3415-ttl-refresh-before-lte-suggestion', + 'sep-3415-ttl-no-rejection-path', + 'sep-3415-unsubscribe-by-key', + 'sep-3415-unsubscribe-empty-result', + 'sep-3415-unsubscribe-unknown-not-found', + 'sep-3415-error-unsupported' +] as const; + +/** Rows no single-principal, single-process run can exercise. */ +const ALWAYS_UNTESTABLE = [ + 'sep-3415-subscribe-auth-required', + 'sep-3415-subscribe-cross-tenant-isolation', + 'sep-3415-subscribe-refresh-replaces-secret', + 'sep-3415-subscribe-refresh-reactivates', + 'sep-3415-ttl-long-grant-retained', + 'sep-3415-ttl-no-expiry-persisted', + 'sep-3415-ttl-no-expiry-gc-terminated' +] as const; + +async function webhookChecks(opts: EventsFixtureOptions) { + const fixture = await startEventsFixture(opts); + try { + const checks = await new EventsWebhookScenario().run( + testContext(fixture.url, DRAFT_PROTOCOL_VERSION) + ); + // Drained so an intentionally malformed response does not trip the global + // vitest hook; these tests assert on the check, not the wire validator. + takeWireViolations(); + return { + checks: new Map(checks.map((c) => [c.id, c])), + /** What the fixture still holds, which is how cleanup is graded. */ + leaked: fixture.liveSubscriptions() + }; + } finally { + await fixture.close(); + } +} + +/** + * A catalog with one webhook type and one that only polls. The second is not + * decoration: `sep-3415-error-unsupported` needs a type whose `delivery` omits + * webhook, and without one it reports untestable. + */ +function webhookFixture( + subscribe: SubscribeBehaviour = {} +): EventsFixtureOptions { + return { + capability: { listChanged: true }, + descriptors: [ + descriptor({ name: 'hook.event', delivery: ['webhook'] }), + descriptor({ name: 'poll.only', delivery: ['poll'] }) + ], + subscribe + }; +} + +describe('the conformant baseline', () => { + test('every gradeable row passes and the untestable ones never read green', async () => { + const { checks, leaked } = await webhookChecks(webhookFixture()); + expect(checks.size).toBe(ALL_ROWS); + for (const id of GRADEABLE) { + expect(checks.get(id)?.status, id).toBe('SUCCESS'); + } + for (const id of ALWAYS_UNTESTABLE) { + expect(checks.get(id)?.details?.untestable, id).toBe(true); + expect(checks.get(id)?.status, id).not.toBe('SKIPPED'); + } + // The scenario promises to leave nothing behind, including on a public + // server. This is the only place that promise is actually checked. + expect(leaked).toEqual([]); + }); + + // kitchen-sink allows two subscriptions per principal per event type, and a + // scenario that held every probe open would grade -32025 instead of the rule + // it was probing. The release-as-you-go discipline is what prevents that, and + // this is the regression test for it. + test('a two-subscription cap changes nothing, because probes are released', async () => { + const { checks } = await webhookChecks( + webhookFixture({ maxSubscriptions: 2 }) + ); + for (const id of GRADEABLE) { + expect(checks.get(id)?.status, id).toBe('SUCCESS'); + } + }); + + test('a cap of one reports the rows it starves as untestable, not passed', async () => { + const { checks } = await webhookChecks( + webhookFixture({ maxSubscriptions: 1 }) + ); + const check = checks.get('sep-3415-subscribe-key-composition'); + expect(check?.details?.untestable).toBe(true); + expect(check?.errorMessage).toContain('-32025'); + }); + + test('a server with no webhook-capable type reports every row untestable', async () => { + const { checks } = await webhookChecks({ + capability: { listChanged: true }, + descriptors: [descriptor()] + }); + expect(checks.size).toBe(ALL_ROWS); + for (const check of checks.values()) { + expect(check.details?.untestable, check.id).toBe(true); + } + }); + + test('a server that declares nothing and serves nothing skips the suite', async () => { + const { checks } = await webhookChecks({ + listError: { code: -32601, message: 'Method not found' } + }); + for (const check of checks.values()) { + expect(check.status).toBe('SKIPPED'); + } + }); + + test('an unimplemented events/subscribe fails rather than warns', async () => { + const { checks } = await webhookChecks( + webhookFixture({ + error: { code: -32601, message: 'Method not found' } + }) + ); + const check = checks.get('sep-3415-subscribe-webhook-only'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('advertises `webhook` delivery'); + }); + + test('a refused subscribe warns and takes the rest with it as untestable', async () => { + const { checks } = await webhookChecks( + webhookFixture({ error: { code: -32024, message: 'Forbidden' } }) + ); + expect(checks.get('sep-3415-subscribe-webhook-only')?.status).toBe( + 'WARNING' + ); + expect(checks.get('sep-3415-unsubscribe-by-key')?.details?.untestable).toBe( + true + ); + }); +}); + +describe('the delivery secret', () => { + test('accepting a subscribe with no secret fails', async () => { + const { checks } = await webhookChecks( + webhookFixture({ acceptMissingSecret: true }) + ); + const check = checks.get('sep-3415-subscribe-secret-required'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('-32602 InvalidParams'); + }); + + test('accepting a secret without the whsec_ prefix fails', async () => { + const { checks } = await webhookChecks( + webhookFixture({ acceptBadPrefix: true }) + ); + expect(checks.get('sep-3415-subscribe-secret-format')?.status).toBe( + 'FAILURE' + ); + }); + + test('accepting a secret under the 24-byte floor fails', async () => { + const { checks } = await webhookChecks( + webhookFixture({ acceptShortSecret: true }) + ); + const check = checks.get('sep-3415-subscribe-secret-rejected'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('accepted'); + }); + + test('rejecting with the wrong code warns and names -32602', async () => { + const { checks } = await webhookChecks( + webhookFixture({ rejectionCode: -32602 + 1 }) + ); + const check = checks.get('sep-3415-subscribe-secret-required'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('-32602 InvalidParams'); + }); +}); + +describe('the callback URL', () => { + // The divergence this row exists for: kitchen-sink accepts `http://` at + // subscribe, deliberately, for demo ergonomics. + test('accepting an http:// callback fails both URL rows off one probe', async () => { + const { checks } = await webhookChecks( + webhookFixture({ acceptHttpUrl: true }) + ); + const rejected = checks.get('sep-3415-subscribe-url-non-https-rejected'); + expect(rejected?.status).toBe('FAILURE'); + expect(rejected?.errorMessage).toContain('`http://`'); + // The requirement and its enforcement are one probe by design, so the + // second row carries the first's verdict rather than a second probe's. + const required = checks.get('sep-3415-subscribe-url-https-required'); + expect(required?.status).toBe('FAILURE'); + expect(required?.details?.gradedBy).toBe( + 'sep-3415-subscribe-url-non-https-rejected' + ); + }); +}); + +describe('TTL negotiation', () => { + test('no-expiry granted against a finite suggestion fails', async () => { + const { checks } = await webhookChecks( + webhookFixture({ nullRefreshAlways: true }) + ); + const check = checks.get('sep-3415-ttl-null-only-when-requested'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('no expiry'); + // With no finite grant there is nothing to compare to the suggestion. + expect( + checks.get('sep-3415-ttl-refresh-before-lte-suggestion')?.details + ?.untestable + ).toBe(true); + }); + + test('no-expiry for an omitted ttlMs fails the default rule', async () => { + const { checks } = await webhookChecks( + webhookFixture({ nullRefreshOnOmitted: true }) + ); + const check = checks.get('sep-3415-ttl-omitted-means-default'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('explicit `ttlMs: null`'); + }); + + test('granting past the suggestion warns', async () => { + const { checks } = await webhookChecks( + webhookFixture({ grantBeyondSuggestion: true }) + ); + const check = checks.get('sep-3415-ttl-refresh-before-lte-suggestion'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('past the suggested'); + }); + + test('a refreshBefore that is not a timestamp fails', async () => { + const { checks } = await webhookChecks( + webhookFixture({ refreshBefore: 'next tuesday' }) + ); + const check = checks.get('sep-3415-ttl-refresh-before-lte-suggestion'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('ISO 8601'); + }); + + test('rejecting a ttlMs instead of clamping it fails, and names which', async () => { + const { checks } = await webhookChecks( + webhookFixture({ rejectTtl: 'long' }) + ); + const check = checks.get('sep-3415-ttl-no-rejection-path'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('30 days'); + expect(check?.errorMessage).not.toContain('1000ms'); + }); +}); + +describe('the subscription key and its id', () => { + test('a fresh id per call fails the idempotent upsert', async () => { + const { checks } = await webhookChecks( + webhookFixture({ nonIdempotentId: true }) + ); + const check = checks.get('sep-3415-subscribe-idempotent-upsert'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('created a second subscription'); + }); + + test('an id that ignores delivery.url fails composition and immutability', async () => { + const { checks } = await webhookChecks( + webhookFixture({ idIgnoresUrl: true }) + ); + const composition = checks.get('sep-3415-subscribe-key-composition'); + expect(composition?.status).toBe('FAILURE'); + expect(composition?.errorMessage).toContain('not part of the key'); + // Same probe, so the immutability row moves with it. + expect(checks.get('sep-3415-subscribe-key-immutable')?.status).toBe( + 'FAILURE' + ); + }); + + test('honouring a caller-supplied id fails id-not-an-input', async () => { + const { checks } = await webhookChecks( + webhookFixture({ idIsAnInput: true }) + ); + const check = checks.get('sep-3415-subscribe-id-not-an-input'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('addressed the subscription'); + }); + + test('omitting id fails its own row, and costs the comparison rows too', async () => { + const { checks } = await webhookChecks(webhookFixture({ omitId: true })); + const check = checks.get('sep-3415-subscribe-id-derived'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('expected a string'); + // Not an independent defect: with no id there is nothing to compare, so + // the key rows fail alongside it rather than reporting green. + expect(checks.get('sep-3415-subscribe-key-composition')?.status).toBe( + 'FAILURE' + ); + }); + + test('a numeric cursor on the subscribe response fails', async () => { + const { checks } = await webhookChecks( + webhookFixture({ result: { cursor: 42 } }) + ); + expect(checks.get('sep-3415-subscribe-response-cursor')?.status).toBe( + 'FAILURE' + ); + }); + + test('a subscribe response with no refreshBefore fails its own row', async () => { + const { checks } = await webhookChecks( + webhookFixture({ omitRefreshBefore: true }) + ); + const check = checks.get('sep-3415-subscribe-refresh-before-present'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('no `refreshBefore` member'); + // The grant row fails too: an absent grant is not a timestamp. Nothing else + // moves. + const failed = GRADEABLE.filter( + (id) => checks.get(id)?.status !== 'SUCCESS' + ); + expect(failed.sort()).toEqual([ + 'sep-3415-subscribe-refresh-before-present', + 'sep-3415-ttl-refresh-before-lte-suggestion' + ]); + // An explicit null is a grant, not an omission. + const nulled = await webhookChecks( + webhookFixture({ nullRefreshAlways: true }) + ); + expect( + nulled.checks.get('sep-3415-subscribe-refresh-before-present')?.status + ).toBe('SUCCESS'); + }); + + test('a string truncated on the subscribe response fails', async () => { + const { checks } = await webhookChecks( + webhookFixture({ result: { truncated: 'no' } }) + ); + expect(checks.get('sep-3415-subscribe-response-truncated')?.status).toBe( + 'FAILURE' + ); + }); +}); + +describe('unsubscribe', () => { + test('refusing to tear down a key the server holds fails', async () => { + const { checks } = await webhookChecks( + webhookFixture({ unsubscribeHeldCode: -32603 }) + ); + const check = checks.get('sep-3415-unsubscribe-by-key'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('-32603'); + // The unknown-key rule is a separate probe and still passes. + expect(checks.get('sep-3415-unsubscribe-unknown-not-found')?.status).toBe( + 'SUCCESS' + ); + }); + + // The divergence this row exists for: kitchen-sink answers success for a key + // it never held, so a client cannot tell teardown from a typo. + test('success for a key never held fails and says why it matters', async () => { + const { checks } = await webhookChecks( + webhookFixture({ unsubscribeUnknownOk: true }) + ); + const check = checks.get('sep-3415-unsubscribe-unknown-not-found'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('typo'); + }); + + test('an unsubscribe result with a stray member fails its own row', async () => { + const { checks } = await webhookChecks( + webhookFixture({ unsubscribeResult: { removed: true } }) + ); + const check = checks.get('sep-3415-unsubscribe-empty-result'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('`removed`'); + expect( + GRADEABLE.filter((id) => checks.get(id)?.status !== 'SUCCESS') + ).toEqual(['sep-3415-unsubscribe-empty-result']); + // resultType and _meta are allowed on an empty Result. + const meta = await webhookChecks( + webhookFixture({ + unsubscribeResult: { resultType: 'complete', _meta: { x: 1 } } + }) + ); + expect(meta.checks.get('sep-3415-unsubscribe-empty-result')?.status).toBe( + 'SUCCESS' + ); + }); + + test('the wrong code for an unknown key warns and names -32023', async () => { + const { checks } = await webhookChecks( + webhookFixture({ unsubscribeUnknownCode: -32602 }) + ); + const check = checks.get('sep-3415-unsubscribe-unknown-not-found'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('-32023 NotFound'); + }); +}); + +describe('two tenants', () => { + const ROW = 'sep-3415-subscribe-cross-tenant-isolation'; + + test('distinct, independent subscriptions pass', async () => { + const { checks } = await webhookChecks({ + ...webhookFixture(), + tenants: {} + }); + expect(checks.get(ROW)?.status).toBe('SUCCESS'); + }); + + test('a key without the principal fails on the shared id', async () => { + const { checks } = await webhookChecks({ + ...webhookFixture(), + tenants: { keyIgnoresPrincipal: true } + }); + expect(checks.get(ROW)?.status).toBe('FAILURE'); + expect(checks.get(ROW)?.errorMessage).toContain('same subscription id'); + }); + + // The defect the row exists for, and the one it could not see while it + // cleaned up by id: a conformant server refuses `id` as input, so nothing + // was removed and the other tenant always "survived". + test('an unsubscribe that removes the other tenant too fails', async () => { + const { checks } = await webhookChecks({ + ...webhookFixture(), + tenants: { unsubscribeIgnoresPrincipal: true } + }); + expect(checks.get(ROW)?.status).toBe('FAILURE'); + expect(checks.get(ROW)?.errorMessage).toContain('also removed'); + }); + + test('an own unsubscribe that fails is untestable, not a pass', async () => { + const { checks } = await webhookChecks({ + ...webhookFixture({ unsubscribeHeldCode: -32603 }), + tenants: { unsubscribeIgnoresPrincipal: true } + }); + expect(checks.get(ROW)?.details?.untestable).toBe(true); + }); +}); + +describe('SEP-3415 codes and resultType', () => { + test('subscribe and unsubscribe without resultType fail only the resultType row', async () => { + const { checks: ok } = await webhookChecks(webhookFixture()); + const { checks: bare } = await webhookChecks({ + ...webhookFixture(), + omitResultType: true + }); + const check = bare.get('sep-3415-result-type-complete'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('events/subscribe'); + for (const [id, c] of ok) { + if (id === 'sep-3415-result-type-complete') continue; + expect(bare.get(id)?.status, id).toBe(c.status); + } + }); + + test("an unknown unsubscribe key answered with the sketch's -32011 is not NotFound", async () => { + const { checks } = await webhookChecks( + webhookFixture({ unsubscribeUnknownCode: -32011 }) + ); + const check = checks.get('sep-3415-unsubscribe-unknown-not-found'); + expect(check?.status).not.toBe('SUCCESS'); + expect(check?.errorMessage).toContain('-32023 NotFound'); + }); +}); + +describe('the unsupported delivery mode', () => { + test('subscribing a poll-only type fails the unsupported row', async () => { + const { checks } = await webhookChecks( + webhookFixture({ acceptNonWebhookType: true }) + ); + const check = checks.get('sep-3415-error-unsupported'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('does not advertise'); + }); + + test('a catalog where every type offers webhook reports it untestable', async () => { + const { checks } = await webhookChecks({ + capability: { listChanged: true }, + descriptors: [descriptor({ name: 'hook.event', delivery: ['webhook'] })] + }); + const check = checks.get('sep-3415-error-unsupported'); + expect(check?.details?.untestable).toBe(true); + expect(check?.errorMessage).toContain('no type to probe'); + }); +}); + +/** + * The durability rows, through a fixture exposing the restart, generation + * and subscription-state controls. The fixture is stateless, so a restart + * leaves the connection working; the generation control is what tells the + * scenario the restart happened. + */ +describe.concurrent('durability across a restart', () => { + const durable = ( + durability: NonNullable + ): EventsFixtureOptions => ({ ...webhookFixture(), durability }); + + test('a store that keeps both grants and GCs the failing one passes all three', async () => { + const { checks, leaked } = await webhookChecks(durable({ gcAfterMs: 300 })); + for (const id of [ + 'sep-3415-ttl-long-grant-retained', + 'sep-3415-ttl-no-expiry-persisted', + 'sep-3415-ttl-no-expiry-gc-terminated' + ]) { + expect( + checks.get(id)?.status, + `${id}: ${checks.get(id)?.errorMessage}` + ).toBe('SUCCESS'); + } + expect(leaked).toEqual([]); + }); + + test('losing the no-expiry subscription on restart fails the persisted row', async () => { + const { checks } = await webhookChecks( + durable({ dropNoExpiryOnRestart: true, gcAfterMs: 300 }) + ); + const check = checks.get('sep-3415-ttl-no-expiry-persisted'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('never refreshes'); + expect(check?.errorMessage).toContain('`absent`'); + const gc = checks.get('sep-3415-ttl-no-expiry-gc-terminated'); + expect(gc?.details?.untestable, gc?.errorMessage).toBe(true); + }); + + test('losing a long finite grant on restart fails the retained row', async () => { + const { checks } = await webhookChecks( + durable({ dropFiniteOnRestart: true }) + ); + expect(checks.get('sep-3415-ttl-long-grant-retained')?.status).toBe( + 'FAILURE' + ); + }); + + test('never dropping a failing no-expiry subscription is a WARNING, since dropping is a MAY', async () => { + const { checks } = await webhookChecks(durable({})); + const check = checks.get('sep-3415-ttl-no-expiry-gc-terminated'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('MAY'); + }); + + test('a restart control that changes nothing leaves the rows untestable', async () => { + const { checks } = await webhookChecks(durable({ restartNoop: true })); + const check = checks.get('sep-3415-ttl-no-expiry-persisted'); + expect(check?.details?.untestable).toBe(true); + expect(check?.errorMessage).toContain('no evidence a restart happened'); + }); +}); diff --git a/src/scenarios/server/events/negative.test.ts b/src/scenarios/server/events/negative.test.ts new file mode 100644 index 00000000..5fadc13d --- /dev/null +++ b/src/scenarios/server/events/negative.test.ts @@ -0,0 +1,583 @@ +import { describe, test, expect } from 'vitest'; +import { testContext } from '../../../connection/testing'; +import { DRAFT_PROTOCOL_VERSION } from '../../../types'; +import { takeWireViolations } from '../../../validation/wire-schema'; +import { EventsDiscoveryScenario } from './discovery'; +import { EventsPollScenario } from './poll'; +import { + descriptor, + occurrence, + pollResult, + startEventsFixture, + type EventsFixtureOptions +} from './negative-fixture'; + +/** + * Negative controls for the MCP Events scenarios. + * + * A passing run against a conformant fixture proves a check does not + * false-positive. It does not prove the check catches anything, which is what + * these tests are for: every assertion below pairs a conformant server with a + * server broken in exactly one way, and asserts the check flips. + * + * The three divergences the suite is expected to find in mcpkit each get a + * test here, so the checks that will report them are known to work before + * anyone reads a red run and wonders whether the harness is wrong: + * `nextPollSeconds` (gap G31), and the two error-code paths behind event-type + * removal (gap G30). + * + * The fixture is a minimal SEP-2575 stateless server built per test rather + * than a checked-in example file, matching the SEP-2640 negative tests. It + * lives in negative-fixture.ts, shared with the controls for the other three + * scenarios: negative-push.test.ts, negative-webhook.test.ts and + * negative-delivery.test.ts. + */ + +type Scenario = EventsDiscoveryScenario | EventsPollScenario; + +async function checksFor(scenario: Scenario, opts: EventsFixtureOptions) { + const fixture = await startEventsFixture(opts); + try { + const checks = await scenario.run( + testContext(fixture.url, DRAFT_PROTOCOL_VERSION) + ); + // Drained so an intentionally malformed response does not trip the global + // vitest hook; these tests assert on the check, not the wire validator. + takeWireViolations(); + return new Map(checks.map((c) => [c.id, c])); + } finally { + await fixture.close(); + } +} + +const discovery = () => new EventsDiscoveryScenario(); +const poll = () => new EventsPollScenario(); + +/** The baseline every negative case is compared against. */ +const CONFORMANT: EventsFixtureOptions = { + capability: { listChanged: true }, + descriptors: [descriptor()], + pollResponses: [pollResult()] +}; + +describe('events capability declaration', () => { + test('an object declaration passes; a boolean one fails rather than skipping', async () => { + const ok = await checksFor(discovery(), CONFORMANT); + expect(ok.get('sep-3415-capability-events-object')?.status).toBe('SUCCESS'); + + const broken = await checksFor(discovery(), { + ...CONFORMANT, + capability: true + }); + const check = broken.get('sep-3415-capability-events-object'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('a boolean'); + }); + + test('a server that declares nothing and serves nothing skips all but the fallback row', async () => { + const checks = await checksFor(discovery(), { + listError: { code: -32601, message: 'Method not found' } + }); + // Every events rule is inapplicable to a server that does not do events, + // except the one saying what such a server answers. It answered it. + expect(checks.get('sep-3415-fallback-method-not-found')?.status).toBe( + 'SUCCESS' + ); + for (const check of checks.values()) { + if (check.id === 'sep-3415-fallback-method-not-found') continue; + expect(check.status, check.id).toBe('SKIPPED'); + } + }); + + test('the fallback row is emitted even when it cannot apply', async () => { + // A row that vanishes on some servers makes pass counts incomparable + // between them, which is why every path emits the whole set. + const checks = await checksFor(discovery(), CONFORMANT); + const check = checks.get('sep-3415-fallback-method-not-found'); + expect(check).toBeDefined(); + expect(check?.status).toBe('SKIPPED'); + expect(check?.errorMessage).toContain('declares the extension'); + }); + + test('answering something other than -32601 fails the fallback row', async () => { + const checks = await checksFor(discovery(), { + listError: { code: -32000, message: 'nope' } + }); + const check = checks.get('sep-3415-fallback-method-not-found'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('-32601 MethodNotFound'); + }); + + test('empty settings pass, and declared settings make the rule inapplicable', async () => { + const empty = await checksFor(discovery(), { + ...CONFORMANT, + capability: {} + }); + expect(empty.get('sep-3415-capability-empty-settings')?.status).toBe( + 'SUCCESS' + ); + + const populated = await checksFor(discovery(), CONFORMANT); + const check = populated.get('sep-3415-capability-empty-settings'); + expect(check?.status).toBe('SKIPPED'); + expect(check?.errorMessage).toContain('listChanged'); + }); + + // The case mcpkit is actually in: events/list answers, but nothing is + // declared, so a client that reads capabilities first never calls it. A + // plain SKIP here would report that as a clean run. + test('serving events/list while declaring nothing fails rather than skipping', async () => { + const checks = await checksFor(discovery(), { + descriptors: [descriptor()] + }); + const cap = checks.get('sep-3415-capability-events-object'); + expect(cap?.status).toBe('FAILURE'); + expect(cap?.errorMessage).toContain('declares no `capabilities.extensions'); + expect(cap?.errorMessage).toContain('unreachable'); + + // And the rest of the scenario is still graded, not abandoned. + expect(checks.get('sep-3415-list-implemented')?.status).toBe('SUCCESS'); + expect(checks.get('sep-3415-descriptor-name')?.status).toBe('SUCCESS'); + }); + + test('the poll scenario likewise grades an undeclared-but-serving server', async () => { + const checks = await checksFor(poll(), { descriptors: [descriptor()] }); + expect(checks.get('sep-3415-poll-implemented')?.status).toBe('SUCCESS'); + + const skipped = await checksFor(poll(), { + listError: { code: -32601, message: 'Method not found' } + }); + expect(skipped.get('sep-3415-poll-implemented')?.status).toBe('SKIPPED'); + }); + + test('a non-boolean listChanged fails', async () => { + const checks = await checksFor(discovery(), { + ...CONFORMANT, + capability: { listChanged: 'yes' } + }); + const check = checks.get('sep-3415-capability-list-changed-flag'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('a string'); + }); +}); + +describe('events/list descriptors', () => { + test('a descriptor missing name fails', async () => { + const checks = await checksFor(discovery(), { + ...CONFORMANT, + descriptors: [descriptor({ name: undefined })] + }); + expect(checks.get('sep-3415-descriptor-name')?.status).toBe('FAILURE'); + }); + + test('an empty delivery array fails; a valid subset passes', async () => { + const ok = await checksFor(discovery(), CONFORMANT); + expect(ok.get('sep-3415-descriptor-delivery-subset')?.status).toBe( + 'SUCCESS' + ); + + const broken = await checksFor(discovery(), { + ...CONFORMANT, + descriptors: [descriptor({ delivery: [] })] + }); + const check = broken.get('sep-3415-descriptor-delivery-subset'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('non-empty'); + }); + + test('a delivery mode outside poll/push/webhook fails', async () => { + const checks = await checksFor(discovery(), { + ...CONFORMANT, + descriptors: [descriptor({ delivery: ['poll', 'carrier-pigeon'] })] + }); + const check = checks.get('sep-3415-descriptor-delivery-subset'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('carrier-pigeon'); + }); + + test('a missing payloadSchema fails', async () => { + const checks = await checksFor(discovery(), { + ...CONFORMANT, + descriptors: [descriptor({ payloadSchema: undefined })] + }); + expect(checks.get('sep-3415-descriptor-payload-schema')?.status).toBe( + 'FAILURE' + ); + }); + + test('an empty catalog reports descriptor checks untestable, never green', async () => { + const checks = await checksFor(discovery(), { + ...CONFORMANT, + descriptors: [] + }); + const check = checks.get('sep-3415-descriptor-name'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('Not testable'); + expect(check?.details?.untestable).toBe(true); + }); + + test('an unimplemented events/list fails and names the capability mismatch', async () => { + const checks = await checksFor(discovery(), { + ...CONFORMANT, + listError: { code: -32601, message: 'Method not found' } + }); + const check = checks.get('sep-3415-list-implemented'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('declares `capabilities.extensions'); + }); +}); + +describe('events error codes', () => { + test('an unknown event name answering -32023 passes, -32026 fails', async () => { + const ok = await checksFor(discovery(), CONFORMANT); + expect(ok.get('sep-3415-error-not-found')?.status).toBe('SUCCESS'); + + const broken = await checksFor(discovery(), { + ...CONFORMANT, + unknownNameCode: -32026 + }); + const check = broken.get('sep-3415-error-not-found'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('-32026 Unsupported` is for'); + }); + + test('a code SEP-3415 does not allocate fails the allocation check', async () => { + const checks = await checksFor(discovery(), { + ...CONFORMANT, + unknownNameCode: -1 + }); + const check = checks.get('sep-3415-error-server-range'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('not one of the five codes'); + }); + + test("the design sketch's -32011 fails NotFound and the allocation check, and nothing else", async () => { + const ok = await checksFor(discovery(), CONFORMANT); + const legacy = await checksFor(discovery(), { + ...CONFORMANT, + unknownNameCode: -32011 + }); + const notFound = legacy.get('sep-3415-error-not-found'); + expect(notFound?.status).toBe('FAILURE'); + expect(notFound?.errorMessage).toContain('expected -32023 NotFound'); + const range = legacy.get('sep-3415-error-server-range'); + expect(range?.status).toBe('FAILURE'); + expect(range?.errorMessage).toContain("design sketch's numbers"); + for (const [id, check] of ok) { + if (id === 'sep-3415-error-not-found') continue; + if (id === 'sep-3415-error-server-range') continue; + expect(legacy.get(id)?.status, id).toBe(check.status); + } + }); +}); + +describe('resultType on events results', () => { + for (const [label, make] of [ + ['events/list', discovery], + ['events/poll', poll] + ] as const) { + test(`${label} without resultType fails only the resultType row`, async () => { + const ok = await checksFor(make(), CONFORMANT); + expect(ok.get('sep-3415-result-type-complete')?.status).toBe('SUCCESS'); + + const bare = await checksFor(make(), { + ...CONFORMANT, + omitResultType: true + }); + const check = bare.get('sep-3415-result-type-complete'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain(label); + for (const [id, c] of ok) { + if (id === 'sep-3415-result-type-complete') continue; + expect(bare.get(id)?.status, id).toBe(c.status); + } + }); + } +}); + +describe('events/poll response shape', () => { + test('nextPollSeconds is caught and named as the pre-rename field', async () => { + const ok = await checksFor(poll(), CONFORMANT); + expect(ok.get('sep-3415-poll-next-poll-ms')?.status).toBe('SUCCESS'); + + // Gap G31: mcpkit still ships this. The check has to name the rename, or a + // reader of the red run cannot tell it from a server that omits the field. + const legacy = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [ + pollResult({ nextPollMs: undefined, nextPollSeconds: 30 }) + ] + }); + const check = legacy.get('sep-3415-poll-next-poll-ms'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('nextPollSeconds'); + expect(check?.errorMessage).toContain('197c32b4'); + }); + + test('a non-array events field fails', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [pollResult({ events: null })] + }); + expect(checks.get('sep-3415-poll-events-array')?.status).toBe('FAILURE'); + }); + + test('a numeric cursor fails the opaque-string check', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [pollResult({ cursor: 42 })] + }); + expect(checks.get('sep-3415-poll-response-cursor')?.status).toBe('FAILURE'); + expect(checks.get('sep-3415-cursor-opaque')?.status).toBe('FAILURE'); + }); + + test('replaying history for a null cursor fails start-from-now', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [pollResult({ events: [occurrence()] })] + }); + const check = checks.get('sep-3415-cursor-null-starts-from-now'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('start from now'); + }); +}); + +describe('events/poll cursor lifecycle', () => { + test('a quiet poll that drops the cursor fails advancement', async () => { + const ok = await checksFor(poll(), CONFORMANT); + expect(ok.get('sep-3415-cursor-advances-when-quiet')?.status).toBe( + 'SUCCESS' + ); + + const broken = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [pollResult(), pollResult({ cursor: 7 })] + }); + expect(broken.get('sep-3415-cursor-advances-when-quiet')?.status).toBe( + 'FAILURE' + ); + }); + + test('a type that flips between a cursor and null warns on consistency', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [pollResult(), pollResult({ cursor: null })] + }); + const check = checks.get('sep-3415-cursor-consistency'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('branch once'); + }); + + test('rejecting a poll that omits cursor fails absent-means-null', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + // Poll order: 1 bootstraps, 2 carries the cursor forward, 3 is the one + // that omits `cursor` entirely. Only the third may fail here. + pollResponses: [ + pollResult(), + pollResult(), + { error: { code: -32602, message: 'InvalidParams: cursor required' } } + ] + }); + const check = checks.get('sep-3415-cursor-absent-equals-null'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('absent cursor means'); + }); + + test('truncated true with no fresh cursor fails', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [ + pollResult(), + pollResult(), + pollResult(), + pollResult({ truncated: true, cursor: null }) + ] + }); + const check = checks.get('sep-3415-truncated-returns-fresh-cursor'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('no fresh position'); + }); +}); + +describe('events/poll error contract', () => { + test('a poll omitting name must not return a result', async () => { + const ok = await checksFor(poll(), CONFORMANT); + expect(ok.get('sep-3415-poll-one-subscription-per-request')?.status).toBe( + 'SUCCESS' + ); + }); + + test('the -32602 error-table row carries the same verdict as the rule', async () => { + const ok = await checksFor(poll(), CONFORMANT); + const row = ok.get('sep-3415-error-invalid-params'); + expect(row?.status).toBe('SUCCESS'); + expect(row?.details?.gradedBy).toBe('sep-3415-poll-invalid-arguments'); + + const broken = await checksFor(poll(), { + ...CONFORMANT, + invalidArgsCode: -32023 + }); + expect(broken.get('sep-3415-error-invalid-params')?.status).toBe('FAILURE'); + }); + + test('wrong-typed arguments answering -32023 fails the invalid-params check', async () => { + const ok = await checksFor(poll(), CONFORMANT); + expect(ok.get('sep-3415-poll-invalid-arguments')?.status).toBe('SUCCESS'); + + const broken = await checksFor(poll(), { + ...CONFORMANT, + invalidArgsCode: -32023 + }); + const check = broken.get('sep-3415-poll-invalid-arguments'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('expected -32602'); + }); + + test('polling a push-only type must answer -32026, not a result', async () => { + const pushOnly = descriptor({ name: 'push.only', delivery: ['push'] }); + const broken = await checksFor(poll(), { + ...CONFORMANT, + descriptors: [descriptor(), pushOnly], + pollByName: { 'push.only': pollResult() } + }); + const check = broken.get('sep-3415-poll-mode-unsupported'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('does not advertise'); + + const ok = await checksFor(poll(), { + ...CONFORMANT, + descriptors: [descriptor(), pushOnly], + pollByName: { + 'push.only': { error: { code: -32026, message: 'Unsupported' } } + } + }); + expect(ok.get('sep-3415-poll-mode-unsupported')?.status).toBe('SUCCESS'); + }); + + test('every type offering poll reports the unsupported-mode check untestable', async () => { + const checks = await checksFor(poll(), CONFORMANT); + const check = checks.get('sep-3415-poll-mode-unsupported'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('Not testable'); + }); +}); + +describe('EventOccurrence shape', () => { + test('a quiet server reports the occurrence checks untestable, never green', async () => { + const checks = await checksFor(poll(), CONFORMANT); + const check = checks.get('sep-3415-occurrence-event-id'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('Not testable'); + }); + + test('a non-ISO timestamp fails', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [ + pollResult({ + events: [occurrence({ timestamp: 'last tuesday' })], + cursor: null + }) + ] + }); + expect(checks.get('sep-3415-occurrence-timestamp')?.status).toBe('FAILURE'); + }); + + test('a repeated eventId within one batch warns', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [ + pollResult({ + events: [occurrence(), occurrence()], + cursor: null + }) + ] + }); + const check = checks.get('sep-3415-occurrence-event-id-from-upstream'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('dedup'); + }); + + test('a missing data object fails', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [ + pollResult({ + events: [occurrence({ data: undefined })], + cursor: null + }) + ] + }); + expect(checks.get('sep-3415-occurrence-data')?.status).toBe('FAILURE'); + }); +}); + +describe('list_changed over subscriptions/listen', () => { + const GATED = 'sep-3415-capability-list-changed-gated'; + const NOTIFIED = 'sep-3415-list-changed-notification'; + const LISTENING: EventsFixtureOptions = { ...CONFORMANT, listen: {} }; + + /** Ids whose status differs between two runs. */ + const flipped = ( + a: Map, + b: Map + ) => [...a.keys()].filter((id) => a.get(id)?.status !== b.get(id)?.status); + + test('a declaring server that honors the opt-in and notifies on a catalog change passes', async () => { + const checks = await checksFor(discovery(), LISTENING); + expect(checks.get(NOTIFIED)?.status).toBe('SUCCESS'); + expect(checks.get(GATED)?.status).toBe('SKIPPED'); + expect(checks.get(GATED)?.errorMessage).toContain('listChanged: true'); + }); + + test('a server that did not declare listChanged leaves the key out and is not asked to notify', async () => { + const checks = await checksFor(discovery(), { + ...LISTENING, + capability: {} + }); + expect(checks.get(GATED)?.status).toBe('SUCCESS'); + expect(checks.get(NOTIFIED)?.status).toBe('SKIPPED'); + }); + + test('echoing eventsListChanged without declaring listChanged fails only the gate', async () => { + const ok = await checksFor(discovery(), { ...LISTENING, capability: {} }); + const broken = await checksFor(discovery(), { + ...LISTENING, + capability: {}, + listen: { echo: 'always' } + }); + expect(broken.get(GATED)?.status).toBe('FAILURE'); + expect(broken.get(GATED)?.errorMessage).toContain('acknowledged'); + expect(flipped(ok, broken)).toEqual([GATED]); + }); + + test('acknowledging the opt-in but never notifying fails only the notification', async () => { + const ok = await checksFor(discovery(), LISTENING); + const broken = await checksFor(discovery(), { + ...LISTENING, + listen: { sendListChanged: false } + }); + expect(broken.get(NOTIFIED)?.status).toBe('FAILURE'); + expect(broken.get(NOTIFIED)?.errorMessage).toContain( + 'no `notifications/events/list_changed`' + ); + expect(flipped(ok, broken)).toEqual([NOTIFIED]); + }); + + test('declaring listChanged and then declining the opt-in fails the notification', async () => { + const checks = await checksFor(discovery(), { + ...LISTENING, + listen: { echo: 'never' } + }); + expect(checks.get(NOTIFIED)?.status).toBe('FAILURE'); + expect(checks.get(NOTIFIED)?.errorMessage).toContain('did not honor'); + }); + + test('a server with no subscriptions/listen reports the notification untestable, naming the method', async () => { + const checks = await checksFor(discovery(), CONFORMANT); + const check = checks.get(NOTIFIED); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('subscriptions/listen'); + }); +}); diff --git a/src/scenarios/server/events/poll.ts b/src/scenarios/server/events/poll.ts new file mode 100644 index 00000000..45bc2192 --- /dev/null +++ b/src/scenarios/server/events/poll.ts @@ -0,0 +1,1295 @@ +/** + * MCP Events — poll delivery, the `EventOccurrence` shape, and cursor + * lifecycle. + * + * One scenario, many checks (per AGENTS.md "fewer scenarios, more checks"). + * Each check's verbatim spec excerpt lives next to its check ID in + * src/seps/sep-3415.yaml. + * + * Poll is the mode a conformance harness can exercise completely. It is + * request/response, it holds no server-side state, and every poll response + * carries the cursor, so cursor advancement and `truncated` are observable + * without waiting for anything to happen upstream. Push needs a live stream + * and webhook needs a reachable callback; both are separate scenarios. + * + * The scenario drives a real quiet-period poll loop: two polls with the cursor + * from the first fed into the second. That is what makes + * `sep-3415-cursor-advances-when-quiet` gradeable against a server with no + * traffic, which is the state a conformance fixture is usually in. + * + * Nothing is hardcoded to a fixture's event names. The scenario picks the + * first descriptor advertising `poll` and works from there; when none does, + * every check reports the unmet prerequisite rather than passing vacuously. + */ + +import { ClientScenario, ConformanceCheck } from '../../../types'; +import type { Connection, RunContext } from '../../../connection'; +import { untestableCheck } from '../../untestable'; +import { + EVENTS_EXTENSION_ID, + EVENTS_POLL_METHOD, + EVENTS_SPEC_REF, + EVENTS_NOT_FOUND, + EVENTS_UNSUPPORTED, + resultTypeCheck, + RESULT_TYPE_ID, + JSONRPC_INVALID_PARAMS, + JSONRPC_METHOD_NOT_FOUND, + type EventDescriptor, + type EventOccurrence, + type EventsPollResult, + declaredEventsCapability, + eventsCheck, + eventsListAll, + eventsPoll, + occurrences, + deliveryModes, + descriptorName, + describeValue, + isObject, + isValidCursor, + isIso8601, + minimalArguments, + firstSupporting, + unknownEventName +} from './helpers'; + +const POLL_IDS = [ + 'sep-3415-poll-implemented', + 'sep-3415-poll-one-subscription-per-request', + 'sep-3415-poll-bootstraps-subscription', + 'sep-3415-poll-events-array', + 'sep-3415-poll-response-cursor', + 'sep-3415-poll-next-poll-ms', + 'sep-3415-poll-next-poll-ms-ignored-when-has-more', + 'sep-3415-poll-has-more', + 'sep-3415-poll-max-events-cap', + 'sep-3415-poll-stateless-request', + 'sep-3415-poll-errors-are-jsonrpc', + 'sep-3415-poll-invalid-arguments', + 'sep-3415-poll-mode-unsupported', + 'sep-3415-removal-poll-not-found' +] as const; + +/** How long to follow the bootstrap cursor waiting for a delivered event. */ +const OCCURRENCE_WAIT_MS = 6000; + +const OCCURRENCE_IDS = [ + 'sep-3415-occurrence-event-id', + 'sep-3415-occurrence-name', + 'sep-3415-occurrence-timestamp', + 'sep-3415-occurrence-data', + 'sep-3415-occurrence-cursor-optional', + 'sep-3415-occurrence-meta-ungoverned', + 'sep-3415-occurrence-event-id-from-upstream' +] as const; + +const CURSOR_IDS = [ + 'sep-3415-cursor-opaque', + 'sep-3415-cursor-null-starts-from-now', + 'sep-3415-cursor-absent-equals-null', + 'sep-3415-cursor-consistency', + 'sep-3415-cursor-advances-when-quiet', + 'sep-3415-max-age-ms-floor', + 'sep-3415-max-age-ms-sets-truncated', + 'sep-3415-max-age-ms-ignored-when-cursor-null', + 'sep-3415-replay-ceiling-sets-truncated', + 'sep-3415-truncated-returns-fresh-cursor', + 'sep-3415-truncated-poll-never-an-error', + 'sep-3415-truncated-false-when-no-replay' +] as const; + +/** + * Pair the `-32602` rule row with the error-table row for the same code. + * + * One probe, two rows: the first says arguments that violate `inputSchema` are + * rejected, the second says the code for a statically invalid request is + * `-32602`. A server gets the same verdict on both because there is nothing to + * tell apart, and `details.gradedBy` says where it came from. + */ +function withErrorCodeRow(ruleCheck: ConformanceCheck): ConformanceCheck[] { + return [ + ruleCheck, + eventsCheck( + 'sep-3415-error-invalid-params', + "`-32602 InvalidParams` — request is statically invalid: arguments don't match the event's inputSchema, the callback `delivery.url` is malformed or non-`https`, or `delivery.secret` is not a valid `whsec_` value.", + ruleCheck.status, + { + errorMessage: ruleCheck.errorMessage, + details: { + gradedBy: 'sep-3415-poll-invalid-arguments', + untestable: ruleCheck.details?.untestable + } + } + ) + ]; +} + +/** + * The error-table row this scenario claims. `-32602` has three provoking cases + * in the document and the suite already fires one of them here, so the row is + * graded off that probe rather than a fourth request, the way the + * https-required row rides its enforcement probe in events-webhook. That + * precedent is named in prose rather than by its slug, so grepping a check id + * still finds only the scenario that emits it. + */ +const ERROR_IDS = ['sep-3415-error-invalid-params'] as const; + +const ALL_IDS = [ + ...POLL_IDS, + ...OCCURRENCE_IDS, + ...CURSOR_IDS, + ...ERROR_IDS, + RESULT_TYPE_ID +]; + +/** Milliseconds of replay to request when probing the `maxAgeMs` floor. */ +const MAX_AGE_PROBE_MS = 300_000; + +function skipAll(reason: string): ConformanceCheck[] { + return ALL_IDS.map((id) => + eventsCheck(id, reason, 'SKIPPED', { errorMessage: reason }) + ); +} + +function untestableAll( + ids: readonly string[], + reason: string, + severity: 'FAILURE' | 'WARNING' = 'FAILURE' +): ConformanceCheck[] { + return ids.map((id) => + untestableCheck(id, id, id, reason, [EVENTS_SPEC_REF], severity) + ); +} + +export class EventsPollScenario implements ClientScenario { + name = 'events-poll'; + readonly source = { extensionId: EVENTS_EXTENSION_ID } as const; + description = `MCP Events: \`events/poll\` delivery, the \`EventOccurrence\` shape, and cursor lifecycle. + +**Methods**: \`events/poll\`, plus \`events/list\` to discover a poll-capable event type + +**Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-3415.yaml): + +- \`sep-3415-poll-implemented\` / \`sep-3415-poll-bootstraps-subscription\` — a poll with \`cursor: null\` succeeds with no prior subscribe step +- \`sep-3415-poll-events-array\` / \`sep-3415-poll-response-cursor\` / \`sep-3415-poll-next-poll-ms\` / \`sep-3415-poll-has-more\` — the four response fields +- \`sep-3415-poll-max-events-cap\` — \`maxEvents\` caps the batch and sets \`hasMore\` when more remain +- \`sep-3415-poll-stateless-request\` — two identical polls are answerable without server-side memory of the first +- \`sep-3415-poll-errors-are-jsonrpc\` / \`sep-3415-removal-poll-not-found\` / \`sep-3415-poll-invalid-arguments\` / \`sep-3415-poll-mode-unsupported\` — the error contract +- \`sep-3415-occurrence-*\` — \`eventId\`, \`name\`, \`timestamp\`, \`data\` required; \`cursor\` and \`_meta\` optional +- \`sep-3415-cursor-*\` — opaqueness, \`null\` means start-from-now, absent means \`null\`, consistency, and advancement during quiet periods +- \`sep-3415-max-age-ms-*\` / \`sep-3415-truncated-*\` — bounding replay and signalling a gap + +**Discovery is dynamic**: a server that neither declares the capability nor implements \`events/list\` SKIPs everything; no poll-capable event type reports the poll checks as untestable. Checks that need a delivered event (the \`EventOccurrence\` shape) are untestable against a quiet server rather than passing vacuously.`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect(); + try { + return await this.checks(conn); + } finally { + await conn.close(); + } + } + + private async checks(conn: Connection): Promise { + const { declared } = await declaredEventsCapability(conn); + const listed = await eventsListAll(conn); + + if ('error' in listed) { + // Undeclared and unimplemented is the one case that legitimately skips: + // the server simply does not do events. Every other shape is graded, + // including the undeclared-but-serving case events-discovery fails on. + if (!declared && listed.error.code === JSONRPC_METHOD_NOT_FOUND) { + return skipAll( + 'Server does not declare the `events` capability and does not implement `events/list`; the extension is optional.' + ); + } + return untestableAll( + ALL_IDS, + `\`events/list\` failed (${listed.error.code} ${listed.error.message}), so no poll-capable event type could be discovered. See the events-discovery scenario.` + ); + } + + const descriptors = listed.descriptors; + const target = firstSupporting(descriptors, 'poll'); + const name = target ? descriptorName(target) : undefined; + + if (!target || !name) { + return untestableAll( + ALL_IDS, + descriptors.length === 0 + ? '`events/list` returned an empty catalog, so no poll-capable event type could be exercised.' + : 'No event type advertises `poll` delivery, so `events/poll` could not be exercised. Poll is optional per event type.' + ); + } + + const args = minimalArguments(target); + if (args === undefined) { + return untestableAll( + ALL_IDS, + `Event type \`${name}\` declares required \`inputSchema\` properties, so the harness cannot construct arguments it is confident the server will accept.` + ); + } + + const checks: ConformanceCheck[] = []; + + // --- The bootstrap poll ---------------------------------------------- + const first = await eventsPoll(conn, { + name, + arguments: args, + cursor: null + }); + if ('error' in first) { + const err = first.error; + checks.push( + eventsCheck( + 'sep-3415-poll-implemented', + '`events/poll` is request/response: client sends `{name, arguments, cursor}`, gets back `{events[], cursor, nextPollMs}`.', + 'FAILURE', + { + errorMessage: `Event type \`${name}\` advertises \`poll\` delivery but \`${EVENTS_POLL_METHOD}\` failed: ${err.code} ${err.message}`, + details: { code: err.code, message: err.message, data: err.data } + } + ) + ); + checks.push( + ...untestableAll( + ALL_IDS.filter((id) => id !== 'sep-3415-poll-implemented'), + `The bootstrap \`${EVENTS_POLL_METHOD}\` for \`${name}\` failed with ${err.code} ${err.message}.` + ) + ); + return checks; + } + + const r1 = first.result; + checks.push(resultTypeCheck([{ method: EVENTS_POLL_METHOD, result: r1 }])); + checks.push( + eventsCheck( + 'sep-3415-poll-implemented', + '`events/poll` is request/response: client sends `{name, arguments, cursor}`, gets back `{events[], cursor, nextPollMs}`.', + 'SUCCESS' + ) + ); + checks.push( + eventsCheck( + 'sep-3415-poll-bootstraps-subscription', + 'No separate subscribe step needed — the first poll with a null cursor bootstraps the subscription.', + 'SUCCESS', + { details: { name } } + ) + ); + + checks.push(...this.responseShapeChecks(r1)); + checks.push(...this.cursorNullChecks(r1)); + + // --- Quiet-period advancement ---------------------------------------- + checks.push(...(await this.quietAdvanceChecks(conn, name, args, r1))); + + // --- maxEvents / hasMore --------------------------------------------- + checks.push(...(await this.maxEventsChecks(conn, name, args))); + + // --- maxAgeMs and truncated ------------------------------------------ + checks.push(...(await this.replayChecks(conn, name, args, r1))); + + // --- EventOccurrence shape ------------------------------------------- + // Not graded off r1: `cursor: null` starts from now, so a conformant + // bootstrap poll is empty. Follow its cursor until something is delivered. + const delivered = await this.pollForOccurrences(conn, name, args, r1); + checks.push(...this.occurrenceChecks(delivered, target)); + + // --- Error contract --------------------------------------------------- + checks.push(...(await this.errorChecks(conn, descriptors, name, args))); + + return checks; + } + + /** The four response fields, graded off the bootstrap poll. */ + private responseShapeChecks(r: EventsPollResult): ConformanceCheck[] { + const out: ConformanceCheck[] = []; + + out.push( + Array.isArray(r.events) + ? eventsCheck( + 'sep-3415-poll-events-array', + 'The poll response carries an `events` array. An empty array means nothing happened.', + 'SUCCESS', + { details: { count: occurrences(r).length } } + ) + : eventsCheck( + 'sep-3415-poll-events-array', + 'The poll response carries an `events` array. An empty array means nothing happened.', + 'FAILURE', + { + errorMessage: `\`events\` is ${describeValue(r.events)}, expected an array (empty when nothing happened).`, + details: { events: r.events } + } + ) + ); + + out.push( + isValidCursor(r.cursor) + ? eventsCheck( + 'sep-3415-poll-response-cursor', + 'The poll response carries `cursor` at the response level, the subscription position after this batch.', + 'SUCCESS', + { details: { cursor: r.cursor ?? null } } + ) + : eventsCheck( + 'sep-3415-poll-response-cursor', + 'The poll response carries `cursor` at the response level, the subscription position after this batch.', + 'FAILURE', + { + errorMessage: `\`cursor\` is ${describeValue(r.cursor)}, expected a string, null, or absent.`, + details: { cursor: r.cursor } + } + ) + ); + + // nextPollMs is the field the 197c32b4 rename introduced. A server still + // emitting nextPollSeconds is the single most likely failure here, so name + // it rather than reporting a generic absence. + const legacy = 'nextPollSeconds' in r; + if (typeof r.nextPollMs === 'number' && Number.isFinite(r.nextPollMs)) { + out.push( + eventsCheck( + 'sep-3415-poll-next-poll-ms', + '`nextPollMs` allows the server to dynamically adjust polling frequency.', + 'SUCCESS', + { details: { nextPollMs: r.nextPollMs } } + ) + ); + } else { + out.push( + eventsCheck( + 'sep-3415-poll-next-poll-ms', + '`nextPollMs` allows the server to dynamically adjust polling frequency.', + 'WARNING', + { + errorMessage: legacy + ? 'Response carries `nextPollSeconds`, the pre-rename field name. Spec commit `197c32b4` (2026-05-10) renamed the duration fields to `nextPollMs`.' + : `\`nextPollMs\` is ${describeValue(r.nextPollMs)}, expected a number of milliseconds.`, + details: { + nextPollMs: r.nextPollMs, + nextPollSeconds: r.nextPollSeconds + } + } + ) + ); + } + + out.push( + r.hasMore === undefined || typeof r.hasMore === 'boolean' + ? eventsCheck( + 'sep-3415-poll-has-more', + '`hasMore` indicates whether additional events are available beyond the returned batch.', + 'SUCCESS', + { details: { hasMore: r.hasMore ?? false } } + ) + : eventsCheck( + 'sep-3415-poll-has-more', + '`hasMore` indicates whether additional events are available beyond the returned batch.', + 'FAILURE', + { + errorMessage: `\`hasMore\` is ${describeValue(r.hasMore)}, expected a boolean.`, + details: { hasMore: r.hasMore } + } + ) + ); + + // `nextPollMs` is ignored when `hasMore` is true, which is only observable + // on a response that actually sets it. + out.push( + r.hasMore === true + ? eventsCheck( + 'sep-3415-poll-next-poll-ms-ignored-when-has-more', + '`nextPollMs` is ignored when `hasMore` is `true`.', + 'SUCCESS', + { details: { hasMore: true, nextPollMs: r.nextPollMs } } + ) + : untestableCheck( + 'sep-3415-poll-next-poll-ms-ignored-when-has-more', + 'sep-3415-poll-next-poll-ms-ignored-when-has-more', + '`nextPollMs` is ignored when `hasMore` is `true`.', + 'Server has no backlog, so no response set `hasMore: true` and the interaction between the two fields could not be observed.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + + return out; + } + + /** What a `cursor: null` bootstrap poll establishes on its own. */ + private cursorNullChecks(r: EventsPollResult): ConformanceCheck[] { + const out: ConformanceCheck[] = []; + const events = occurrences(r); + + out.push( + events.length === 0 + ? eventsCheck( + 'sep-3415-cursor-null-starts-from-now', + 'Passing `cursor: null` means "start from now." No historical events are replayed.', + 'SUCCESS', + { details: { returned: 0 } } + ) + : eventsCheck( + 'sep-3415-cursor-null-starts-from-now', + 'Passing `cursor: null` means "start from now." No historical events are replayed.', + 'FAILURE', + { + errorMessage: `A poll with \`cursor: null\` returned ${events.length} event(s); "start from now" replays nothing.`, + details: { count: events.length } + } + ) + ); + + out.push( + isValidCursor(r.cursor) + ? eventsCheck( + 'sep-3415-cursor-opaque', + 'Cursors are opaque strings managed by the server, representing a position in the event stream.', + 'SUCCESS', + { + details: { + cursorType: + r.cursor === null || r.cursor === undefined + ? 'null' + : 'string' + } + } + ) + : eventsCheck( + 'sep-3415-cursor-opaque', + 'Cursors are opaque strings managed by the server, representing a position in the event stream.', + 'FAILURE', + { + errorMessage: `\`cursor\` is ${describeValue(r.cursor)}; a cursor is an opaque string (or null when the type has no replay).`, + details: { cursor: r.cursor } + } + ) + ); + + return out; + } + + /** + * Poll twice with the cursor from the first response and check the server + * answers the second without being told anything it was not told the first + * time. + * + * This grades three rows at once: the cursor advances (or stays put + * legitimately) during a quiet period, the request is self-contained, and an + * omitted `cursor` field is accepted as `null`. + */ + private async quietAdvanceChecks( + conn: Connection, + name: string, + args: Record, + first: EventsPollResult + ): Promise { + const out: ConformanceCheck[] = []; + const cursor = first.cursor; + const noReplay = cursor === null || cursor === undefined; + + const second = await eventsPoll(conn, { + name, + arguments: args, + ...(noReplay ? {} : { cursor }) + }); + + if ('error' in second) { + const reason = `A follow-up \`${EVENTS_POLL_METHOD}\` carrying the cursor from the first response failed: ${second.error.code} ${second.error.message}`; + out.push( + eventsCheck( + 'sep-3415-poll-stateless-request', + 'Each poll request is self-contained: the server does not need to remember previous poll requests to answer them.', + 'FAILURE', + { errorMessage: reason, details: { code: second.error.code } } + ) + ); + out.push( + ...untestableAll( + [ + 'sep-3415-cursor-advances-when-quiet', + 'sep-3415-cursor-consistency' + ], + reason + ) + ); + out.push( + ...untestableAll( + ['sep-3415-cursor-absent-equals-null'], + reason, + 'FAILURE' + ) + ); + return out; + } + + const r2 = second.result; + + out.push( + eventsCheck( + 'sep-3415-poll-stateless-request', + 'Each poll request is self-contained: the server does not need to remember previous poll requests to answer them.', + 'SUCCESS', + { details: { secondPollAccepted: true } } + ) + ); + + // Cursor advancement during a quiet period. A server with no traffic may + // legitimately return the same cursor — the requirement is that the + // response carries one at all, so the client's persisted position does not + // go stale. An event type with no replay carries null both times, which is + // equally conformant. + out.push( + isValidCursor(r2.cursor) + ? eventsCheck( + 'sep-3415-cursor-advances-when-quiet', + "Every poll response carries `cursor`, including when `events: []`, so the client's persisted cursor advances during quiet periods.", + 'SUCCESS', + { + details: { + first: cursor ?? null, + second: r2.cursor ?? null, + advanced: (r2.cursor ?? null) !== (cursor ?? null) + } + } + ) + : eventsCheck( + 'sep-3415-cursor-advances-when-quiet', + "Every poll response carries `cursor`, including when `events: []`, so the client's persisted cursor advances during quiet periods.", + 'FAILURE', + { + errorMessage: `Follow-up poll returned \`cursor\` as ${describeValue(r2.cursor)}; a quiet poll must still carry a position.`, + details: { cursor: r2.cursor } + } + ) + ); + + // Consistency: a type that returns a cursor once returns one always. + const firstNull = cursor === null || cursor === undefined; + const secondNull = r2.cursor === null || r2.cursor === undefined; + out.push( + firstNull === secondNull + ? eventsCheck( + 'sep-3415-cursor-consistency', + 'An event type that ever returns a non-null cursor SHOULD always do so, and one that returns `null` SHOULD always return `null`.', + 'SUCCESS', + { details: { replaySupported: !firstNull } } + ) + : eventsCheck( + 'sep-3415-cursor-consistency', + 'An event type that ever returns a non-null cursor SHOULD always do so, and one that returns `null` SHOULD always return `null`.', + 'WARNING', + { + errorMessage: `Event type \`${name}\` returned ${firstNull ? 'null' : 'a cursor'} then ${secondNull ? 'null' : 'a cursor'}; clients branch once at subscribe time on this.`, + details: { first: cursor ?? null, second: r2.cursor ?? null } + } + ) + ); + + // Absent means null: the second poll omitted `cursor` entirely when the + // type has no replay, and the server answered anyway. + out.push( + noReplay + ? eventsCheck( + 'sep-3415-cursor-absent-equals-null', + 'An absent `cursor` field MUST be treated identically to an explicit `cursor: null`; a receiver MUST NOT fail because it is missing.', + 'SUCCESS', + { details: { omittedCursorAccepted: true } } + ) + : await this.absentCursorCheck(conn, name, args) + ); + + return out; + } + + /** Probe "absent means null" directly by omitting the field. */ + private async absentCursorCheck( + conn: Connection, + name: string, + args: Record + ): Promise { + const id = 'sep-3415-cursor-absent-equals-null'; + const description = + 'An absent `cursor` field MUST be treated identically to an explicit `cursor: null`; a receiver MUST NOT fail because it is missing.'; + const probe = await eventsPoll(conn, { name, arguments: args }); + if ('error' in probe) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `A poll omitting \`cursor\` was rejected with ${probe.error.code} ${probe.error.message}; an absent cursor means "start from now", not a malformed request.`, + details: { code: probe.error.code, message: probe.error.message } + }); + } + return eventsCheck(id, description, 'SUCCESS', { + details: { returned: occurrences(probe.result).length } + }); + } + + /** `maxEvents` caps the batch; `hasMore` reports whether more remain. */ + private async maxEventsChecks( + conn: Connection, + name: string, + args: Record + ): Promise { + const id = 'sep-3415-poll-max-events-cap'; + const description = + '`maxEvents` is an optional cap on the number of events returned. If more are available, the server returns a partial batch with an intermediate cursor and sets `hasMore: true`.'; + + const probe = await eventsPoll(conn, { + name, + arguments: args, + cursor: null, + maxEvents: 1 + }); + + if ('error' in probe) { + return [ + eventsCheck(id, description, 'FAILURE', { + errorMessage: `A poll carrying \`maxEvents: 1\` was rejected with ${probe.error.code} ${probe.error.message}; \`maxEvents\` is an optional request field, not an error.`, + details: { code: probe.error.code, message: probe.error.message } + }) + ]; + } + + const returned = occurrences(probe.result).length; + if (returned > 1) { + return [ + eventsCheck(id, description, 'FAILURE', { + errorMessage: `A poll with \`maxEvents: 1\` returned ${returned} events.`, + details: { returned } + }) + ]; + } + + return [ + eventsCheck(id, description, 'SUCCESS', { + details: { returned, hasMore: probe.result.hasMore ?? false } + }) + ]; + } + + /** + * `maxAgeMs` and `truncated`. + * + * Against a quiet server the floor never advances past the cursor, so the + * two `truncated`-setting rows cannot be driven to true. What is gradeable + * everywhere: the field is accepted, it is ignored when the cursor is null, + * `truncated` is a boolean rather than an error, and when it is true the + * response still carries a fresh cursor. + */ + private async replayChecks( + conn: Connection, + name: string, + args: Record, + bootstrap: EventsPollResult + ): Promise { + const out: ConformanceCheck[] = []; + const noReplay = + bootstrap.cursor === null || bootstrap.cursor === undefined; + + const withMaxAge = await eventsPoll(conn, { + name, + arguments: args, + cursor: bootstrap.cursor ?? null, + maxAgeMs: MAX_AGE_PROBE_MS + }); + + if ('error' in withMaxAge) { + const reason = `A poll carrying \`maxAgeMs\` was rejected with ${withMaxAge.error.code} ${withMaxAge.error.message}.`; + out.push( + eventsCheck( + 'sep-3415-max-age-ms-floor', + 'All three modes accept an optional `maxAgeMs` alongside `cursor`; the server begins replay from whichever is later, the cursor or `now − maxAgeMs`.', + 'FAILURE', + { errorMessage: reason, details: { code: withMaxAge.error.code } } + ) + ); + out.push( + ...untestableAll( + [ + 'sep-3415-max-age-ms-sets-truncated', + 'sep-3415-replay-ceiling-sets-truncated', + 'sep-3415-truncated-returns-fresh-cursor', + 'sep-3415-truncated-poll-never-an-error', + 'sep-3415-truncated-false-when-no-replay', + 'sep-3415-max-age-ms-ignored-when-cursor-null' + ], + reason, + 'WARNING' + ) + ); + return out; + } + + const r = withMaxAge.result; + out.push( + eventsCheck( + 'sep-3415-max-age-ms-floor', + 'All three modes accept an optional `maxAgeMs` alongside `cursor`; the server begins replay from whichever is later, the cursor or `now − maxAgeMs`.', + 'SUCCESS', + { details: { maxAgeMs: MAX_AGE_PROBE_MS } } + ) + ); + + // truncated is a response field, never an error. A server that rejected + // the maxAgeMs poll outright already failed above. + const truncated = r.truncated; + out.push( + truncated === undefined || typeof truncated === 'boolean' + ? eventsCheck( + 'sep-3415-truncated-poll-never-an-error', + 'For poll, `truncated` appears in the result body. Never a JSON-RPC error.', + 'SUCCESS', + { details: { truncated: truncated ?? false } } + ) + : eventsCheck( + 'sep-3415-truncated-poll-never-an-error', + 'For poll, `truncated` appears in the result body. Never a JSON-RPC error.', + 'FAILURE', + { + errorMessage: `\`truncated\` is ${describeValue(truncated)}, expected a boolean.`, + details: { truncated } + } + ) + ); + + // When truncated is true the response must still carry a servable cursor. + out.push( + truncated === true + ? isValidCursor(r.cursor) && r.cursor !== null && r.cursor !== undefined + ? eventsCheck( + 'sep-3415-truncated-returns-fresh-cursor', + 'The server resets to a position it can serve from and returns that position as the fresh `cursor` alongside `truncated: true`.', + 'SUCCESS', + { details: { cursor: r.cursor } } + ) + : eventsCheck( + 'sep-3415-truncated-returns-fresh-cursor', + 'The server resets to a position it can serve from and returns that position as the fresh `cursor` alongside `truncated: true`.', + 'FAILURE', + { + errorMessage: `\`truncated: true\` was returned with \`cursor\` as ${describeValue(r.cursor)}; the client has no fresh position to persist.`, + details: { cursor: r.cursor } + } + ) + : untestableCheck( + 'sep-3415-truncated-returns-fresh-cursor', + 'sep-3415-truncated-returns-fresh-cursor', + 'The server resets to a position it can serve from and returns that position as the fresh `cursor` alongside `truncated: true`.', + 'No probe produced `truncated: true`, so the fresh-cursor obligation could not be observed. Driving it requires a stale cursor the harness cannot mint against an opaque cursor space.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + + for (const id of [ + 'sep-3415-max-age-ms-sets-truncated', + 'sep-3415-replay-ceiling-sets-truncated' + ]) { + out.push( + untestableCheck( + id, + id, + id, + "Requires a cursor older than the `maxAgeMs` floor or the server's replay ceiling. Cursors are opaque, so the harness cannot mint a stale one, and a quiet fixture has no history to fall out of.", + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + } + + out.push( + noReplay + ? truncated === true + ? eventsCheck( + 'sep-3415-truncated-false-when-no-replay', + 'For event types that do not support replay (`cursor` is always `null`), `truncated` SHOULD be `false`.', + 'WARNING', + { + errorMessage: `Event type \`${name}\` returns \`cursor: null\` (no replay) but set \`truncated: true\`; there is no position to have advanced past.`, + details: { truncated } + } + ) + : eventsCheck( + 'sep-3415-truncated-false-when-no-replay', + 'For event types that do not support replay (`cursor` is always `null`), `truncated` SHOULD be `false`.', + 'SUCCESS', + { details: { truncated: truncated ?? false } } + ) + : eventsCheck( + 'sep-3415-truncated-false-when-no-replay', + 'For event types that do not support replay (`cursor` is always `null`), `truncated` SHOULD be `false`.', + 'SKIPPED', + { + errorMessage: `Event type \`${name}\` supports replay, so this rule does not apply to it.` + } + ) + ); + + // maxAgeMs is ignored when cursor is null. Observable as "the server does + // not replay history in response to it". + const nullCursorMaxAge = await eventsPoll(conn, { + name, + arguments: args, + cursor: null, + maxAgeMs: MAX_AGE_PROBE_MS + }); + const id = 'sep-3415-max-age-ms-ignored-when-cursor-null'; + const description = + '`maxAgeMs` is ignored when `cursor` is `null` (null already means "now").'; + if ('error' in nullCursorMaxAge) { + out.push( + eventsCheck(id, description, 'FAILURE', { + errorMessage: `A poll with \`cursor: null\` and \`maxAgeMs\` was rejected with ${nullCursorMaxAge.error.code} ${nullCursorMaxAge.error.message}.`, + details: { code: nullCursorMaxAge.error.code } + }) + ); + } else { + const replayed = occurrences(nullCursorMaxAge.result).length; + out.push( + replayed === 0 + ? eventsCheck(id, description, 'SUCCESS', { details: { replayed } }) + : eventsCheck(id, description, 'FAILURE', { + errorMessage: `A poll with \`cursor: null\` and \`maxAgeMs: ${MAX_AGE_PROBE_MS}\` replayed ${replayed} event(s); \`maxAgeMs\` is ignored when the cursor is null.`, + details: { replayed } + }) + ); + } + + return out; + } + + /** + * The `EventOccurrence` field contract. + * + * Only gradeable against a response that actually carried an event. A quiet + * fixture reports the whole group as untestable rather than passing an empty + * array through seven shape checks, which would read as green. + */ + /** + * Poll forward from the bootstrap cursor, waiting `nextPollMs` between + * attempts (clamped to 250ms-2s), until a batch carries events or + * OCCURRENCE_WAIT_MS elapses. Returns the last result either way, so a quiet + * server still reports the occurrence checks as untestable. + */ + private async pollForOccurrences( + conn: Connection, + name: string, + args: Record, + bootstrap: EventsPollResult + ): Promise { + let last = bootstrap; + const deadline = Date.now() + OCCURRENCE_WAIT_MS; + while (occurrences(last).length === 0 && Date.now() < deadline) { + const hint = typeof last.nextPollMs === 'number' ? last.nextPollMs : 1000; + const wait = Math.min(Math.max(hint, 250), 2000, deadline - Date.now()); + await new Promise((resolve) => setTimeout(resolve, wait)); + const next = await eventsPoll(conn, { + name, + arguments: args, + cursor: last.cursor ?? null + }); + if ('error' in next) return last; + last = next.result; + } + return last; + } + + private occurrenceChecks( + r: EventsPollResult, + descriptor: EventDescriptor + ): ConformanceCheck[] { + const events = occurrences(r); + if (events.length === 0) { + return untestableAll( + OCCURRENCE_IDS, + 'No poll returned an event, so the `EventOccurrence` shape could not be validated. The fixture needs a diagnostic event type that emits on demand.' + ); + } + + const out: ConformanceCheck[] = []; + const label = (e: EventOccurrence, i: number) => + typeof e.eventId === 'string' ? `\`${e.eventId}\`` : `events[${i}]`; + + const field = ( + id: string, + description: string, + severity: 'FAILURE' | 'WARNING', + predicate: (e: EventOccurrence) => string | undefined + ) => { + for (const [i, e] of events.entries()) { + const problem = predicate(e); + if (problem) { + out.push( + eventsCheck(id, description, severity, { + errorMessage: `${label(e, i)}: ${problem}`, + details: { occurrence: e } + }) + ); + return; + } + } + out.push( + eventsCheck(id, description, 'SUCCESS', { + details: { occurrencesChecked: events.length } + }) + ); + }; + + field( + 'sep-3415-occurrence-event-id', + '`eventId` (string) is required on every `EventOccurrence`: a stable identifier for deduplication.', + 'FAILURE', + (e) => + typeof e.eventId === 'string' && e.eventId.length > 0 + ? undefined + : `\`eventId\` is ${describeValue(e.eventId)}, expected a non-empty string.` + ); + + field( + 'sep-3415-occurrence-name', + '`name` (string) is required on every `EventOccurrence`: the event type name.', + 'FAILURE', + (e) => + typeof e.name === 'string' && e.name === descriptorName(descriptor) + ? undefined + : `\`name\` is ${JSON.stringify(e.name)}, expected \`${descriptorName(descriptor)}\`.` + ); + + field( + 'sep-3415-occurrence-timestamp', + '`timestamp` (string, ISO 8601) is required on every `EventOccurrence`: when the event occurred.', + 'FAILURE', + (e) => + isIso8601(e.timestamp) + ? undefined + : `\`timestamp\` is ${JSON.stringify(e.timestamp)}, expected an ISO 8601 instant.` + ); + + field( + 'sep-3415-occurrence-data', + "`data` (object) is required on every `EventOccurrence`: payload conforming to the event type's `payloadSchema`.", + 'FAILURE', + (e) => + isObject(e.data) + ? undefined + : `\`data\` is ${describeValue(e.data)}, expected an object.` + ); + + field( + 'sep-3415-occurrence-cursor-optional', + '`cursor` on an `EventOccurrence` is optional; poll carries the cursor at the response level.', + 'FAILURE', + (e) => + isValidCursor(e.cursor) + ? undefined + : `\`cursor\` is ${describeValue(e.cursor)}, expected a string, null, or absent.` + ); + + field( + 'sep-3415-occurrence-meta-ungoverned', + '`_meta` is reserved for protocol/extension metadata and is not governed by `payloadSchema`.', + 'WARNING', + (e) => + e._meta === undefined || isObject(e._meta) + ? undefined + : `\`_meta\` is ${describeValue(e._meta)}, expected an object when present.` + ); + + // Whether an eventId came from upstream is not observable from one run; + // what is observable is that ids are distinct within a batch, which a + // per-delivery counter would violate. + const ids = events + .map((e) => e.eventId) + .filter((v): v is string => typeof v === 'string'); + out.push( + new Set(ids).size === ids.length + ? eventsCheck( + 'sep-3415-occurrence-event-id-from-upstream', + "The server SHOULD use the upstream's stable event identifier as `eventId` so the same upstream event carries the same id across delivery paths.", + 'SUCCESS', + { details: { distinct: new Set(ids).size, total: ids.length } } + ) + : eventsCheck( + 'sep-3415-occurrence-event-id-from-upstream', + "The server SHOULD use the upstream's stable event identifier as `eventId` so the same upstream event carries the same id across delivery paths.", + 'WARNING', + { + errorMessage: + 'A single batch repeated an `eventId`, so the value cannot be an upstream-stable identifier and client-side dedup would drop distinct events.', + details: { ids } + } + ) + ); + + return out; + } + + /** The poll error contract: unknown name, bad arguments, unsupported mode. */ + private async errorChecks( + conn: Connection, + descriptors: EventDescriptor[], + name: string, + args: Record + ): Promise { + const out: ConformanceCheck[] = []; + + // Unknown name. + const unknown = unknownEventName(); + const probe = await eventsPoll(conn, { + name: unknown, + arguments: {}, + cursor: null + }); + const notFoundDesc = + 'A poll against a name the server does not serve returns `-32023 NotFound`.'; + const jsonRpcDesc = + 'Errors are returned as a standard JSON-RPC error response for the request; there is no partial-success model.'; + + if ('error' in probe) { + out.push( + eventsCheck( + 'sep-3415-poll-errors-are-jsonrpc', + jsonRpcDesc, + 'SUCCESS', + { + details: { code: probe.error.code } + } + ) + ); + out.push( + probe.error.code === EVENTS_NOT_FOUND + ? eventsCheck( + 'sep-3415-removal-poll-not-found', + notFoundDesc, + 'SUCCESS', + { + details: { code: probe.error.code, probedName: unknown } + } + ) + : eventsCheck( + 'sep-3415-removal-poll-not-found', + notFoundDesc, + 'FAILURE', + { + errorMessage: `Unknown event name answered ${probe.error.code}, expected ${EVENTS_NOT_FOUND} NotFound.`, + details: { + code: probe.error.code, + message: probe.error.message + } + } + ) + ); + } else { + const msg = `A poll for unknown event name \`${unknown}\` returned a result instead of an error.`; + out.push( + eventsCheck( + 'sep-3415-poll-errors-are-jsonrpc', + jsonRpcDesc, + 'FAILURE', + { + errorMessage: msg, + details: { result: probe.result } + } + ) + ); + out.push( + eventsCheck( + 'sep-3415-removal-poll-not-found', + notFoundDesc, + 'FAILURE', + { + errorMessage: msg + } + ) + ); + } + + // One subscription per request: `name` identifies it, so a poll without + // one is not a request the server can answer. + const noName = await eventsPoll(conn, { arguments: {}, cursor: null }); + const oneSubDesc = + 'Each `events/poll` request carries one subscription, identified by `name`.'; + out.push( + 'error' in noName + ? noName.error.code === JSONRPC_INVALID_PARAMS + ? eventsCheck( + 'sep-3415-poll-one-subscription-per-request', + oneSubDesc, + 'SUCCESS', + { details: { code: noName.error.code } } + ) + : eventsCheck( + 'sep-3415-poll-one-subscription-per-request', + oneSubDesc, + 'WARNING', + { + errorMessage: `A poll omitting \`name\` answered ${noName.error.code}, expected ${JSONRPC_INVALID_PARAMS} InvalidParams.`, + details: { + code: noName.error.code, + message: noName.error.message + } + } + ) + : eventsCheck( + 'sep-3415-poll-one-subscription-per-request', + oneSubDesc, + 'FAILURE', + { + errorMessage: + 'A poll omitting `name` returned a result; each request carries exactly one subscription and `name` is what identifies it.', + details: { result: noName.result } + } + ) + ); + + // Invalid arguments. Only probeable when the schema constrains something. + out.push( + ...(await this.invalidArgumentsCheck(conn, descriptors, name, args)) + ); + + // A delivery mode the event type does not offer. + out.push(await this.unsupportedModeCheck(conn, descriptors)); + + return out; + } + + /** + * Send arguments the descriptor's `inputSchema` cannot accept. + * + * Only constructible when the schema declares a typed property: a fully open + * schema has no invalid value to send, and inventing one would grade the + * server on a rule the schema never stated. + */ + private async invalidArgumentsCheck( + conn: Connection, + descriptors: EventDescriptor[], + name: string, + _args: Record + ): Promise { + const id = 'sep-3415-poll-invalid-arguments'; + const description = + "A poll whose `arguments` do not match the event's `inputSchema` returns `-32602 InvalidParams`."; + + const target = descriptors.find((d) => descriptorName(d) === name); + const schema = target?.inputSchema; + const props = + isObject(schema) && isObject(schema.properties) + ? schema.properties + : undefined; + const typed = props + ? Object.entries(props).find( + ([, v]) => + isObject(v) && + typeof v.type === 'string' && + ['string', 'boolean', 'integer', 'number'].includes(v.type) + ) + : undefined; + + if (!typed) { + return withErrorCodeRow( + untestableCheck( + id, + id, + description, + `Event type \`${name}\` declares no typed \`inputSchema\` property, so no argument value can be known-invalid against it.`, + [EVENTS_SPEC_REF], + 'FAILURE' + ) + ); + } + + const [prop, spec] = typed; + const propType = (spec as Record).type as string; + // A value of the wrong JSON type for the declared one. + const wrongValue = propType === 'string' ? 12345 : 'not-a-valid-value'; + + const probe = await eventsPoll(conn, { + name, + arguments: { [prop]: wrongValue }, + cursor: null + }); + + if (!('error' in probe)) { + return withErrorCodeRow( + eventsCheck(id, description, 'FAILURE', { + errorMessage: `A poll sending \`${prop}: ${JSON.stringify(wrongValue)}\` against a declared \`${propType}\` returned a result instead of ${JSONRPC_INVALID_PARAMS} InvalidParams.`, + details: { property: prop, declaredType: propType, sent: wrongValue } + }) + ); + } + + return withErrorCodeRow( + probe.error.code === JSONRPC_INVALID_PARAMS + ? eventsCheck(id, description, 'SUCCESS', { + details: { property: prop, declaredType: propType } + }) + : eventsCheck(id, description, 'FAILURE', { + errorMessage: `Arguments violating \`inputSchema\` answered ${probe.error.code}, expected ${JSONRPC_INVALID_PARAMS} InvalidParams.`, + details: { + property: prop, + declaredType: propType, + code: probe.error.code, + message: probe.error.message + } + }) + ); + } + + /** Poll an event type whose `delivery` omits `poll`. */ + private async unsupportedModeCheck( + conn: Connection, + descriptors: EventDescriptor[] + ): Promise { + const id = 'sep-3415-poll-mode-unsupported'; + const description = + 'A poll against an event type whose `delivery` does not list `poll` returns `-32026 Unsupported`.'; + + const nonPoll = descriptors.find( + (d) => + descriptorName(d) !== undefined && !deliveryModes(d).includes('poll') + ); + if (!nonPoll) { + return untestableCheck( + id, + id, + description, + 'Every event type the server offers advertises `poll` delivery, so there is no event type to probe the unsupported-mode path with.', + [EVENTS_SPEC_REF], + 'FAILURE' + ); + } + + const name = descriptorName(nonPoll)!; + const probe = await eventsPoll(conn, { + name, + arguments: minimalArguments(nonPoll) ?? {}, + cursor: null + }); + + if (!('error' in probe)) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `Event type \`${name}\` does not advertise \`poll\` delivery (\`delivery\` is ${JSON.stringify(nonPoll.delivery)}) but \`${EVENTS_POLL_METHOD}\` returned a result.`, + details: { name, delivery: deliveryModes(nonPoll) } + }); + } + + return probe.error.code === EVENTS_UNSUPPORTED + ? eventsCheck(id, description, 'SUCCESS', { + details: { name, code: probe.error.code, data: probe.error.data } + }) + : eventsCheck(id, description, 'FAILURE', { + errorMessage: `Polling \`${name}\`, which does not offer poll delivery, answered ${probe.error.code}, expected ${EVENTS_UNSUPPORTED} Unsupported.`, + details: { + name, + delivery: deliveryModes(nonPoll), + code: probe.error.code, + message: probe.error.message + } + }); + } +} + +/** Exported for the negative tests, which assert the full emitted set. */ +export const EVENTS_POLL_CHECK_IDS = ALL_IDS; diff --git a/src/scenarios/server/events/push.ts b/src/scenarios/server/events/push.ts new file mode 100644 index 00000000..194b3234 --- /dev/null +++ b/src/scenarios/server/events/push.ts @@ -0,0 +1,1662 @@ +/** + * MCP Events: push delivery over a long-lived `events/stream` request. + * + * Scored against the merged design sketch on `main` of + * modelcontextprotocol/experimental-ext-triggers-events. Each check's verbatim + * excerpt lives beside its id in src/seps/sep-3415.yaml. + * + * Two things shape what this scenario can grade. + * + * The heartbeat is a MUST with a SHOULD of "at least every 30 seconds", so a + * conformant server may say nothing for 30s. The observation window therefore + * has to outlast that, which is why the default is 35s and why the scenario + * needs `--timeout 60000` rather than the runner's 30s default. A shorter + * window would report a compliant slow-heartbeat server as broken, which is + * worse than taking the time. + * + * Several rows need a server doing something no protocol request can ask for: + * an upstream failure (`stream-error-is-recoverable`), a retention gap + * (`stream-gap-resends-active`), a termination (`stream-terminated-*`), or a + * server-initiated close (`stream-final-result-*`). + * + * A fixture MAY expose diagnostic controls as ordinary tools, and this scenario + * calls the two that are safe to fire mid-stream: neither ends the + * subscription, so the rows below still see the heartbeats and deliveries they + * grade. Termination is not fired, because it is terminal for the source and + * these scenarios share one fixture process, so terminating here would poison + * whatever runs next. + * + * A server with no controls is unaffected and keeps reporting untestable with + * the missing prerequisite named, per src/scenarios/untestable.ts, rather than + * passing vacuously against a server that simply never did it. + */ + +import { ClientScenario, ConformanceCheck } from '../../../types'; +import type { Connection, RunContext } from '../../../connection'; +import { untestableCheck } from '../../untestable'; +import { + EVENTS_ACTIVE_NOTIFICATION, + EVENTS_EXTENSION_ID, + EVENTS_CONTROL_YIELD_ERROR, + EVENTS_CONTROL_YIELD_GAP, + EVENTS_CONTROL_TERMINATE, + EVENTS_CONTROL_QUOTA, + EVENTS_CONTROL_CATALOG, + askControl, + deliveryModes, + type EventDescriptor, + hasControl, + fireControl, + extensionsOf, + EVENTS_ERROR_NOTIFICATION, + EVENTS_EVENT_NOTIFICATION, + EVENTS_HEARTBEAT_NOTIFICATION, + EVENTS_SPEC_REF, + EVENTS_STREAM_METHOD, + resultTypeCheck, + RESULT_TYPE_ID, + EVENTS_TERMINATED_NOTIFICATION, + EVENTS_NOT_FOUND, + EVENTS_RESOURCE_EXHAUSTED, + EVENTS_UNSUPPORTED, + JSONRPC_METHOD_NOT_FOUND, + SUBSCRIPTION_ID_META, + describeValue, + descriptorName, + eventsCheck, + eventsListAll, + firstSupporting, + isIso8601, + isObject, + isValidCursor, + minimalArguments +} from './helpers'; +import { + openEventStream, + type StreamNotification, + type StreamSession +} from './stream'; + +/** How long to watch an idle stream. Must outlast the document's 30s SHOULD. */ +const WATCH_MS = Number(process.env.EVENTS_PUSH_WATCH_MS ?? 35000); + +/** How long to wait for the subscription confirmation before grading it. */ +const ACTIVE_MS = 3000; +// How long to wait for the terminated frame after firing the control. Generous +// against ACTIVE_MS because the control travels on its own connection and the +// fanout is asynchronous. +const TERMINATE_MS = 5000; + +/** Slack on the 30s heartbeat SHOULD, for scheduling and network jitter. */ +const HEARTBEAT_TOLERANCE_MS = 2000; + +const STREAM_IDS = [ + 'sep-3415-stream-implemented', + 'sep-3415-stream-error-before-open', + 'sep-3415-stream-mode-unsupported', + 'sep-3415-stream-active-confirmation', + 'sep-3415-stream-subscription-id-meta', + 'sep-3415-stream-event-notification', + 'sep-3415-stream-error-is-recoverable', + 'sep-3415-stream-terminated-ends-subscription', + 'sep-3415-stream-gap-resends-active', + 'sep-3415-stream-heartbeat-required', + 'sep-3415-stream-heartbeat-carries-cursor', + 'sep-3415-stream-heartbeat-interval', + 'sep-3415-stream-heartbeat-not-sse-comment', + 'sep-3415-stream-final-result-shape', + 'sep-3415-stream-final-result-timing', + RESULT_TYPE_ID, + 'sep-3415-stream-cancel-stops-delivery', + 'sep-3415-stream-exempt-from-concurrency-cap', + 'sep-3415-stream-carries-only-event-notifications' +] as const; + +/** + * The error-table row this scenario claims. `-32025` is only reached when a + * server actually refuses for a quota, and the concurrency probe is the one + * place the suite asks for more than one of anything, so the row is graded there + * and reported untestable on a server that never hits a limit. + */ +const ERROR_IDS = ['sep-3415-error-resource-exhausted'] as const; + +/** + * The rows about a catalog that changes under a live stream (SEP-3415 § Event + * type removal and breaking changes). Reachable only through the + * `events_conformance_catalog` control, which mints a disposable type per probe + * so removing or retyping it poisons nothing another row relies on. Severity + * follows each row: the termination signal is a SHOULD, the two error shapes + * and the additive MUST NOT are FAILURE. + */ +const CATALOG_ROWS = { + terminates: { + id: 'sep-3415-removal-terminates-subscriptions', + severity: 'WARNING', + description: + "The server SHOULD end them with each mode's termination signal: `notifications/events/terminated` on push streams, and a `terminated` envelope to webhook subscriptions." + }, + notFound: { + id: 'sep-3415-removal-error-not-found', + severity: 'FAILURE', + description: + '`NotFound` with `data: {"kind": "event"}` when the type was removed.' + }, + schemaChanged: { + id: 'sep-3415-removal-error-schema-changed', + severity: 'FAILURE', + description: + '`Unsupported` with `data: {"feature": "payloadSchema" | "inputSchema", "reason": "schema_changed"}` when it was changed in place.' + }, + additive: { + id: 'sep-3415-removal-additive-no-terminate', + severity: 'FAILURE', + description: 'Purely additive changes MUST NOT terminate subscriptions.' + } +} as const; +type CatalogRow = (typeof CATALOG_ROWS)[keyof typeof CATALOG_ROWS]; +const CATALOG_IDS = Object.values(CATALOG_ROWS).map((r) => r.id); + +/** How long an additive change gets to (wrongly) end a stream. */ +const ADDITIVE_WATCH_MS = 2000; + +/** The `error` of a `notifications/events/terminated`, if it carries one. */ +function terminatedError( + n: StreamNotification | undefined +): { code?: unknown; data?: Record } | undefined { + const err = n?.params?.error; + if (!isObject(err)) return undefined; + return { + code: err.code, + data: isObject(err.data) ? (err.data as Record) : undefined + }; +} + +function catalogUntestable(row: CatalogRow, reason: string): ConformanceCheck { + return untestableCheck( + row.id, + row.id, + row.description, + reason, + [EVENTS_SPEC_REF], + row.severity + ); +} + +/** + * Graded by poll too, where a replay-capable target makes it not applicable. + * Push can reach a type without replay whenever one offers push, so it grades + * the gap signal there: `truncated` SHOULD stay false with no position to have + * advanced past. + */ +const NO_REPLAY_ID = 'sep-3415-truncated-false-when-no-replay'; +const NO_REPLAY_DESCRIPTION = + 'For event types that do not support replay (`cursor` is always `null`), `truncated` SHOULD be `false`.'; + +/** How long to watch a no-replay stream after asking for a gap. */ +const NO_REPLAY_WATCH_MS = 1500; + +function untestableAll( + ids: readonly string[], + reason: string, + severity: 'FAILURE' | 'WARNING' = 'FAILURE' +): ConformanceCheck[] { + return ids.map((id) => + untestableCheck(id, id, id, reason, [EVENTS_SPEC_REF], severity) + ); +} + +function skipAll(reason: string): ConformanceCheck[] { + return [...STREAM_IDS, ...ERROR_IDS, NO_REPLAY_ID].map((id) => + eventsCheck(id, id, 'SKIPPED', { errorMessage: reason }) + ); +} + +export class EventsPushScenario implements ClientScenario { + name = 'events-push'; + readonly source = { extensionId: EVENTS_EXTENSION_ID } as const; + description = `MCP Events: push delivery over a long-lived \`events/stream\` request. + +**Methods**: \`events/stream\`, plus \`events/list\` to discover an event type advertising \`push\` delivery + +**Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-3415.yaml): + +- \`sep-3415-stream-implemented\` / \`sep-3415-stream-error-before-open\` — a valid subscription opens a stream; an invalid one answers a JSON-RPC error and opens nothing +- \`sep-3415-stream-active-confirmation\` / \`sep-3415-stream-subscription-id-meta\` — the \`notifications/events/active\` confirmation, and the parent request id echoed on every notification +- \`sep-3415-stream-event-notification\` / \`sep-3415-stream-carries-only-event-notifications\` — events arrive as \`notifications/events/event\`, and nothing else rides the stream +- \`sep-3415-stream-heartbeat-*\` — the keepalive is required, carries a cursor, arrives at least every 30s, and is a \`data:\` frame rather than an SSE comment +- \`sep-3415-stream-exempt-from-concurrency-cap\` — concurrent streams stay open together +- \`sep-3415-stream-cancel-stops-delivery\` / \`sep-3415-stream-final-result-*\` — cancellation and the empty final result + +**This scenario needs \`--timeout 60000\`.** It watches an idle stream for ${WATCH_MS / 1000}s, because a server may heartbeat as slowly as every 30s and still be conformant. Override the window with \`EVENTS_PUSH_WATCH_MS\`. + +**Untestable rather than green**: an upstream failure, a retention gap, a termination and a server-initiated close cannot be provoked from the client side, so those rows name the missing prerequisite instead of passing against a server that simply never did it.`; + + async run(ctx: RunContext): Promise { + const checks = await this.runStreams(ctx); + if (checks.length > 0 && checks.every((c) => c.status === 'SKIPPED')) { + for (const id of CATALOG_IDS) { + checks.push( + eventsCheck(id, id, 'SKIPPED', { + errorMessage: checks[0].errorMessage + }) + ); + } + } else { + checks.push(...(await this.catalogChecks(ctx))); + } + // Every path reports the no-replay row, so runs stay comparable: the early + // returns above the stream never reach the probe that grades it. + if (!checks.some((c) => c.id === NO_REPLAY_ID)) { + checks.push( + untestableCheck( + NO_REPLAY_ID, + NO_REPLAY_ID, + NO_REPLAY_DESCRIPTION, + 'No stream was opened, so no event type without replay was probed.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + } + return checks; + } + + private async runStreams(ctx: RunContext): Promise { + const conn = await ctx.connect(); + let declared = false; + try { + const capabilities = await conn.discover(); + const caps = isObject(capabilities.capabilities) + ? capabilities.capabilities + : {}; + declared = extensionsOf(caps)[EVENTS_EXTENSION_ID] !== undefined; + + const listed = await eventsListAll(conn); + if ('error' in listed) { + if (!declared && listed.error.code === JSONRPC_METHOD_NOT_FOUND) { + return skipAll( + 'Server does not declare the `events` capability and does not implement `events/list`; the extension is optional.' + ); + } + return untestableAll( + [...STREAM_IDS, ...ERROR_IDS], + `\`events/list\` failed (${listed.error.code} ${listed.error.message}), so no push-capable event type could be discovered. See the events-discovery scenario.` + ); + } + + const target = firstSupporting(listed.descriptors, 'push'); + const name = target ? descriptorName(target) : undefined; + if (!target || !name) { + // A server whose types decline push is exactly where the + // unsupported-mode row has something to grade. + return dedupe([ + await this.unsupportedModeCheck(ctx, listed.descriptors), + ...untestableAll( + [...STREAM_IDS, ...ERROR_IDS], + listed.descriptors.length === 0 + ? '`events/list` returned an empty catalog, so no push-capable event type could be exercised.' + : 'No event type advertises `push` delivery, so `events/stream` could not be exercised. Push is optional per event type.' + ) + ]); + } + + // Probe for the diagnostic controls here, on the connection that is + // already open. A server without them must pay nothing for the question: + // doing this later, around the stream, cost two round trips inside the + // observation window and was enough to miss a termination arriving at + // 250ms. + const controls = { + error: await hasControl(conn, EVENTS_CONTROL_YIELD_ERROR), + gap: await hasControl(conn, EVENTS_CONTROL_YIELD_GAP), + terminate: await hasControl(conn, EVENTS_CONTROL_TERMINATE) + }; + + const args = minimalArguments(target); + if (args === undefined) { + return untestableAll( + [...STREAM_IDS, ...ERROR_IDS], + `Event type \`${name}\` declares required \`inputSchema\` properties the harness cannot satisfy from the schema, so no stream could be opened.` + ); + } + + const quota = await this.readQuota(conn); + return await this.streamChecks( + ctx, + name, + args, + controls, + listed.descriptors, + quota + ); + } finally { + await conn.close(); + } + } + + private async streamChecks( + ctx: RunContext, + name: string, + args: Record, + controls: { error: boolean; gap: boolean; terminate: boolean }, + descriptors: EventDescriptor[], + quota?: QuotaReport + ): Promise { + const checks: ConformanceCheck[] = []; + const session = await openEventStream( + ctx.serverUrl, + ctx.specVersion, + { name, arguments: args, cursor: null }, + { openTimeoutMs: ACTIVE_MS } + ); + + try { + // --- The stream opened at all --------------------------------------- + if (!session.contentType?.includes('text/event-stream')) { + const err = session.error; + checks.push( + eventsCheck( + 'sep-3415-stream-implemented', + 'Push delivery uses a long-lived `events/stream` request — one per subscription, a standard JSON-RPC request with an `id`.', + 'FAILURE', + { + errorMessage: err + ? `Event type \`${name}\` advertises \`push\` delivery but \`${EVENTS_STREAM_METHOD}\` answered ${err.code} ${err.message} instead of opening a stream.` + : `\`${EVENTS_STREAM_METHOD}\` answered HTTP ${session.status} with content-type ${describeValue(session.contentType)}, expected an SSE stream.`, + details: { status: session.status, error: err } + } + ) + ); + checks.push( + // Independent of the target stream, so graded before the rest are + // stamped untestable; dedupe keeps the first per id. + await this.unsupportedModeCheck(ctx, descriptors), + ...untestableAll( + [...STREAM_IDS, ...ERROR_IDS].filter( + (id) => id !== 'sep-3415-stream-implemented' + ), + `No stream was opened for \`${name}\`, so nothing on it could be observed.` + ) + ); + checks.push(...(await this.errorBeforeOpenChecks(ctx, args))); + return dedupe(checks); + } + + checks.push( + eventsCheck( + 'sep-3415-stream-implemented', + 'Push delivery uses a long-lived `events/stream` request — one per subscription, a standard JSON-RPC request with an `id`.', + 'SUCCESS', + { details: { name, status: session.status } } + ) + ); + + // --- Confirmation ---------------------------------------------------- + const active = await session.waitFor( + (n) => n.method === EVENTS_ACTIVE_NOTIFICATION, + ACTIVE_MS + ); + checks.push(this.activeCheck(active, session)); + + // --- Provoke the conditions a healthy server never produces ---------- + // An upstream failure and a retention gap are both things the protocol + // gives a client no way to ask for, so against a server with no + // diagnostic controls these rows watch, see nothing, and report + // untestable. A fixture that registers the controls gets them graded. + // Fired inside the observation window so the frames land in this + // session's notification list alongside everything else. + // + // Neither control ends the subscription, which is what makes them safe + // to fire here: the stream stays open and the rows below still see the + // heartbeats and deliveries they grade. + // + // Fired over their own connection rather than the streaming one, which is + // also the more faithful simulation: an upstream failure does not arrive + // as a request from the subscriber watching for it. The signals fan out + // to every live subscriber of the event type, so they reach this session + // regardless. A server with no controls opens no connection here. + if (controls.error || controls.gap) { + const control = await ctx.connect(); + try { + if (controls.error) { + await fireControl(control, EVENTS_CONTROL_YIELD_ERROR, name); + } + if (controls.gap) { + await fireControl(control, EVENTS_CONTROL_YIELD_GAP, name); + } + } finally { + await control.close(); + } + } + + // --- Watch the stream ------------------------------------------------ + // One window serves every timing row: heartbeats, delivered events, and + // whatever else the server chooses to put on the stream. + await session.settle(WATCH_MS); + + checks.push(...this.notificationChecks(session, name, controls)); + checks.push(...this.heartbeatChecks(session)); + + // --- Cancellation ------------------------------------------------------ + const beforeCancel = session.notifications.length; + await session.cancel(); + await sleep(500); + checks.push( + session.notifications.length === beforeCancel + ? eventsCheck( + 'sep-3415-stream-cancel-stops-delivery', + 'On cancellation the server MUST stop delivering events and release any associated resources.', + 'SUCCESS', + { + details: { + note: 'Aborting the request stream is the Streamable HTTP cancel; no further frames were read.', + notifications: beforeCancel + } + } + ) + : eventsCheck( + 'sep-3415-stream-cancel-stops-delivery', + 'On cancellation the server MUST stop delivering events and release any associated resources.', + 'FAILURE', + { + errorMessage: `${session.notifications.length - beforeCancel} further notification(s) arrived after the request stream was aborted.` + } + ) + ); + + // Opportunistic, and pushed first so dedupe keeps it: a server that + // closed this stream itself has already answered the final-result rows, + // and does not need the control path to provoke a second close. + checks.push(...this.finalResultChecks(session, false)); + + checks.push( + ...(await this.terminateChecks( + ctx, + descriptors, + name, + controls.terminate + )) + ); + // Ahead of the concurrency probe, whose own -32025 grading is the + // fallback for a server without the control; dedupe keeps the first. + if (quota) checks.push(await this.quotaCheck(ctx, quota, descriptors)); + checks.push(...(await this.concurrencyChecks(ctx, name, args))); + checks.push(await this.noReplayGapCheck(ctx, descriptors, controls.gap)); + checks.push(...(await this.errorBeforeOpenChecks(ctx, args))); + checks.push(await this.unsupportedModeCheck(ctx, descriptors)); + return dedupe(checks); + } finally { + await session.cancel(); + } + } + + /** + * `truncated-false-when-no-replay` on push: ask for a gap on a type whose + * stream confirms with `cursor: null`, and check no `active` frame arrives + * with `truncated: true`. + * + * Replay support is read off the stream's own `active`, the push analogue + * of poll reading `cursor: null` off its result, since `events/list` has no + * field that declares it. Each push type is opened in turn until one + * confirms with a null cursor; a catalog where every type replays makes the + * rule not applicable, as it is on poll. + */ + private async noReplayGapCheck( + ctx: RunContext, + descriptors: EventDescriptor[], + hasGap: boolean + ): Promise { + const untestable = (reason: string) => + untestableCheck( + NO_REPLAY_ID, + NO_REPLAY_ID, + NO_REPLAY_DESCRIPTION, + reason, + [EVENTS_SPEC_REF], + 'WARNING' + ); + if (!hasGap) { + return untestable( + `A gap cannot be provoked from the client side. Needs a fixture exposing the \`${EVENTS_CONTROL_YIELD_GAP}\` control for an event type without replay.` + ); + } + + let chosen: { name: string; session: StreamSession } | undefined; + for (const d of descriptors) { + const name = descriptorName(d); + const args = minimalArguments(d); + if (!name || args === undefined || !deliveryModes(d).includes('push')) + continue; + const session = await openEventStream( + ctx.serverUrl, + ctx.specVersion, + { name, arguments: args, cursor: null }, + { openTimeoutMs: ACTIVE_MS } + ); + const active = await session.waitFor( + (n) => n.method === EVENTS_ACTIVE_NOTIFICATION, + ACTIVE_MS + ); + if (active && active.params.cursor === null) { + chosen = { name, session }; + break; + } + await session.cancel(); + } + if (!chosen) { + return eventsCheck(NO_REPLAY_ID, NO_REPLAY_DESCRIPTION, 'SKIPPED', { + errorMessage: + 'Every push-capable event type confirmed its stream with a non-null cursor, so each supports replay and this rule does not apply.' + }); + } + + const { name, session } = chosen; + try { + const control = await ctx.connect(); + let fired: boolean; + try { + fired = await fireControl(control, EVENTS_CONTROL_YIELD_GAP, name); + } finally { + await control.close(); + } + if (!fired) { + return untestable( + `\`${name}\` confirms with \`cursor: null\`, but \`${EVENTS_CONTROL_YIELD_GAP}\` declined to signal a gap on it.` + ); + } + await session.settle(NO_REPLAY_WATCH_MS); + const truncated = session.notifications.find( + (n) => + n.method === EVENTS_ACTIVE_NOTIFICATION && n.params.truncated === true + ); + return truncated + ? eventsCheck(NO_REPLAY_ID, NO_REPLAY_DESCRIPTION, 'WARNING', { + errorMessage: `Event type \`${name}\` confirms its stream with \`cursor: null\` (no replay), and after a gap it sent \`${EVENTS_ACTIVE_NOTIFICATION}\` with \`truncated: true\`; there is no position to have advanced past.`, + details: { name, frame: truncated.params } + }) + : eventsCheck(NO_REPLAY_ID, NO_REPLAY_DESCRIPTION, 'SUCCESS', { + details: { name, watchedMs: NO_REPLAY_WATCH_MS } + }); + } finally { + await session.cancel(); + } + } + + /** `notifications/events/active {cursor, truncated, _meta.subscriptionId}`. */ + private activeCheck( + active: { params: Record } | undefined, + session: StreamSession + ): ConformanceCheck { + const description = + 'The server confirms the subscription with `notifications/events/active {cursor, truncated, _meta.subscriptionId}`.'; + if (!active) { + return eventsCheck( + 'sep-3415-stream-active-confirmation', + description, + 'FAILURE', + { + errorMessage: `No \`${EVENTS_ACTIVE_NOTIFICATION}\` arrived within ${ACTIVE_MS}ms of the stream opening.`, + details: { + notifications: session.notifications.map((n) => n.method) + } + } + ); + } + const problems: string[] = []; + if (!isValidCursor(active.params.cursor)) { + problems.push( + `\`cursor\` is ${describeValue(active.params.cursor)}, expected a string or null` + ); + } + if ( + active.params.truncated !== undefined && + typeof active.params.truncated !== 'boolean' + ) { + problems.push( + `\`truncated\` is ${describeValue(active.params.truncated)}, expected a boolean` + ); + } + return problems.length === 0 + ? eventsCheck( + 'sep-3415-stream-active-confirmation', + description, + 'SUCCESS', + { details: { params: active.params } } + ) + : eventsCheck( + 'sep-3415-stream-active-confirmation', + description, + 'FAILURE', + { + errorMessage: problems.join('; '), + details: { params: active.params } + } + ); + } + + /** What rode the stream, and whether every frame was routable. */ + private notificationChecks( + session: StreamSession, + name: string, + controls: { error: boolean; gap: boolean; terminate: boolean } + ): ConformanceCheck[] { + const out: ConformanceCheck[] = []; + const notifications = session.notifications; + + // Every notifications/events/* message carries the parent request id. + const missingMeta = notifications.filter((n) => { + const meta = isObject(n.params._meta) ? n.params._meta : {}; + return meta[SUBSCRIPTION_ID_META] !== session.requestId; + }); + out.push( + missingMeta.length === 0 && notifications.length > 0 + ? eventsCheck( + 'sep-3415-stream-subscription-id-meta', + 'Every `notifications/events/*` message carries the JSON-RPC `id` of the parent `events/stream` request in `params._meta["io.modelcontextprotocol/subscriptionId"]`.', + 'SUCCESS', + { details: { checked: notifications.length } } + ) + : notifications.length === 0 + ? untestableCheck( + 'sep-3415-stream-subscription-id-meta', + 'sep-3415-stream-subscription-id-meta', + 'Every `notifications/events/*` message carries the parent request id in `_meta`.', + 'The stream delivered no notifications, so no message could be checked for the correlation id.', + [EVENTS_SPEC_REF] + ) + : eventsCheck( + 'sep-3415-stream-subscription-id-meta', + 'Every `notifications/events/*` message carries the JSON-RPC `id` of the parent `events/stream` request in `params._meta["io.modelcontextprotocol/subscriptionId"]`.', + 'FAILURE', + { + errorMessage: `${missingMeta.length} of ${notifications.length} notification(s) did not carry \`${SUBSCRIPTION_ID_META}\` = ${session.requestId}.`, + details: { + offending: missingMeta + .slice(0, 3) + .map((n) => ({ method: n.method, meta: n.params._meta })) + } + } + ) + ); + + // The stream carries only notifications/events/*. + const foreign = notifications.filter( + (n) => !n.method.startsWith('notifications/events/') + ); + out.push( + foreign.length === 0 + ? eventsCheck( + 'sep-3415-stream-carries-only-event-notifications', + 'The `events/stream` response carries only `notifications/events/*` messages; it is not a general server-to-client channel.', + 'SUCCESS', + { details: { checked: notifications.length } } + ) + : eventsCheck( + 'sep-3415-stream-carries-only-event-notifications', + 'The `events/stream` response carries only `notifications/events/*` messages; it is not a general server-to-client channel.', + 'FAILURE', + { + errorMessage: `Non-event messages rode the stream: ${[...new Set(foreign.map((n) => n.method))].join(', ')}.` + } + ) + ); + + // Delivered events. + const events = notifications.filter( + (n) => n.method === EVENTS_EVENT_NOTIFICATION + ); + if (events.length === 0) { + out.push( + untestableCheck( + 'sep-3415-stream-event-notification', + 'sep-3415-stream-event-notification', + 'Events are delivered as `notifications/events/event` whose params are an `EventOccurrence`.', + `No event was delivered for \`${name}\` in ${WATCH_MS}ms. The fixture needs an event type that emits while the stream is open.`, + [EVENTS_SPEC_REF] + ) + ); + } else { + const bad = events.filter((n) => { + const p = n.params; + return ( + typeof p.eventId !== 'string' || + typeof p.name !== 'string' || + !isIso8601(p.timestamp) || + !isObject(p.data) + ); + }); + out.push( + bad.length === 0 + ? eventsCheck( + 'sep-3415-stream-event-notification', + 'Events are delivered as `notifications/events/event` whose params are an `EventOccurrence`.', + 'SUCCESS', + { details: { delivered: events.length } } + ) + : eventsCheck( + 'sep-3415-stream-event-notification', + 'Events are delivered as `notifications/events/event` whose params are an `EventOccurrence`.', + 'FAILURE', + { + errorMessage: `${bad.length} of ${events.length} event notification(s) were not a valid EventOccurrence (eventId, name, timestamp and data are required).`, + details: { first: bad[0]?.params } + } + ) + ); + } + + // Rows that need the server to do something the harness cannot ask for. + const errorNotes = notifications.filter( + (n) => n.method === EVENTS_ERROR_NOTIFICATION + ); + out.push( + errorNotes.length === 0 + ? untestableCheck( + 'sep-3415-stream-error-is-recoverable', + 'sep-3415-stream-error-is-recoverable', + '`notifications/events/error` reports a recoverable failure; the subscription remains active.', + controls.error + ? `The \`${EVENTS_CONTROL_YIELD_ERROR}\` control was called for \`${name}\` but no \`${EVENTS_ERROR_NOTIFICATION}\` arrived, so the recovery path could not be observed.` + : `No upstream failure occurred during the run, and the harness cannot provoke one over the protocol. Needs a fixture exposing the \`${EVENTS_CONTROL_YIELD_ERROR}\` control.`, + [EVENTS_SPEC_REF] + ) + : eventsCheck( + 'sep-3415-stream-error-is-recoverable', + '`notifications/events/error` reports a recoverable failure; the subscription remains active and the server retries and resumes.', + session.open ? 'SUCCESS' : 'FAILURE', + { + errorMessage: session.open + ? undefined + : 'The stream closed after `notifications/events/error`, but only `notifications/events/terminated` ends a subscription.', + details: { errors: errorNotes.length } + } + ) + ); + + // A server that terminates on its own during the main window is graded + // here and the control path below never runs for these ids, since dedupe + // is first-wins. Nothing is emitted when it does not: terminateChecks owns + // the untestable branch, because it is the one that knows whether a + // control existed and whether a spare event type was available. + const terminated = notifications.filter( + (n) => n.method === EVENTS_TERMINATED_NOTIFICATION + ); + if (terminated.length > 0) { + out.push( + eventsCheck( + 'sep-3415-stream-terminated-ends-subscription', + 'Only `notifications/events/terminated` ends the subscription.', + 'SUCCESS', + { details: { terminated: terminated.length } } + ) + ); + } + + const actives = notifications.filter( + (n) => n.method === EVENTS_ACTIVE_NOTIFICATION + ); + const gapActive = actives.slice(1).find((n) => n.params.truncated === true); + out.push( + gapActive + ? eventsCheck( + 'sep-3415-stream-gap-resends-active', + 'A gap is not an error — the server sends a fresh `notifications/events/active {cursor:, truncated:true}` and continues delivering.', + 'SUCCESS', + { details: { params: gapActive.params } } + ) + : untestableCheck( + 'sep-3415-stream-gap-resends-active', + 'sep-3415-stream-gap-resends-active', + 'A gap is signalled by a fresh `notifications/events/active` with `truncated: true`, not an error.', + controls.gap + ? `The \`${EVENTS_CONTROL_YIELD_GAP}\` control was called for \`${name}\` but no second \`active\` carrying \`truncated: true\` arrived.` + : `No retention gap occurred during the run, and the harness cannot force one from the client side. Needs a fixture exposing the \`${EVENTS_CONTROL_YIELD_GAP}\` control.`, + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + + return out; + } + + /** The four heartbeat rows, all graded off the same watch window. */ + private heartbeatChecks(session: StreamSession): ConformanceCheck[] { + const out: ConformanceCheck[] = []; + const beats = session.notifications.filter( + (n) => n.method === EVENTS_HEARTBEAT_NOTIFICATION + ); + + // A window shorter than the 30s cadence the document allows cannot tell a + // silent server from a slow one, so it reports the missing prerequisite + // rather than failing a server that may be conformant. + const windowCoversCadence = WATCH_MS > 30000; + out.push( + beats.length > 0 + ? eventsCheck( + 'sep-3415-stream-heartbeat-required', + 'The server MUST send periodic keepalive messages on the push stream so the client can distinguish "nothing to send" from "connection is dead."', + 'SUCCESS', + { details: { beats: beats.length, windowMs: WATCH_MS } } + ) + : windowCoversCadence + ? eventsCheck( + 'sep-3415-stream-heartbeat-required', + 'The server MUST send periodic keepalive messages on the push stream so the client can distinguish "nothing to send" from "connection is dead."', + 'FAILURE', + { + errorMessage: `No \`${EVENTS_HEARTBEAT_NOTIFICATION}\` arrived in ${WATCH_MS}ms, which outlasts the 30s cadence the document asks for.`, + details: { + windowMs: WATCH_MS, + sawSseComments: session.sseComments.length + } + } + ) + : untestableCheck( + 'sep-3415-stream-heartbeat-required', + 'sep-3415-stream-heartbeat-required', + 'The server MUST send periodic keepalive messages on the push stream.', + `No heartbeat arrived, but the watch window was ${WATCH_MS}ms and the document allows a 30s cadence, so a conformant server could look identical. Re-run with EVENTS_PUSH_WATCH_MS above 30000.`, + [EVENTS_SPEC_REF] + ) + ); + + if (beats.length === 0) { + out.push( + ...untestableAll( + [ + 'sep-3415-stream-heartbeat-carries-cursor', + 'sep-3415-stream-heartbeat-interval' + ], + `No heartbeat arrived in ${WATCH_MS}ms, so its contents and cadence could not be observed.` + ) + ); + } else { + const badCursor = beats.filter((n) => !isValidCursor(n.params.cursor)); + out.push( + badCursor.length === 0 + ? eventsCheck( + 'sep-3415-stream-heartbeat-carries-cursor', + 'The heartbeat carries `cursor`, the position the server has checked up to; it is `null` for event types that do not support replay.', + 'SUCCESS', + { details: { beats: beats.length } } + ) + : eventsCheck( + 'sep-3415-stream-heartbeat-carries-cursor', + 'The heartbeat carries `cursor`, the position the server has checked up to; it is `null` for event types that do not support replay.', + 'FAILURE', + { + errorMessage: `${badCursor.length} heartbeat(s) carried a \`cursor\` that was neither a string nor null (first: ${describeValue(badCursor[0]?.params.cursor)}).` + } + ) + ); + + // The gap the client would see: stream open to first beat, then between + // beats. A single beat still bounds the wait the client endured. + const marks = [0, ...beats.map((b) => b.atMs)]; + const gaps = marks.slice(1).map((m, i) => m - marks[i]); + const worst = Math.max(...gaps); + // A server that times its heartbeat at exactly 30s lands a few ms over + // once scheduling and the network have had their say, and reporting that + // as a missed SHOULD is the harness being pedantic rather than the server + // being late. kitchen-sink measured 30005ms. + out.push( + worst <= 30000 + HEARTBEAT_TOLERANCE_MS + ? eventsCheck( + 'sep-3415-stream-heartbeat-interval', + 'The server SHOULD send a heartbeat at least every 30 seconds.', + 'SUCCESS', + { details: { worstGapMs: worst, beats: beats.length } } + ) + : eventsCheck( + 'sep-3415-stream-heartbeat-interval', + 'The server SHOULD send a heartbeat at least every 30 seconds.', + 'WARNING', + { + errorMessage: `Longest silence was ${worst}ms, over the 30s the document asks for (with ${HEARTBEAT_TOLERANCE_MS}ms of tolerance).`, + details: { gapsMs: gaps } + } + ) + ); + } + + out.push( + session.sseComments.length === 0 + ? eventsCheck( + 'sep-3415-stream-heartbeat-not-sse-comment', + 'On Streamable HTTP the heartbeat is an SSE `data:` frame; the SSE comment form (`: keepalive`) is not used since it cannot carry cursor state.', + 'SUCCESS', + { details: { comments: 0 } } + ) + : eventsCheck( + 'sep-3415-stream-heartbeat-not-sse-comment', + 'On Streamable HTTP the heartbeat is an SSE `data:` frame; the SSE comment form (`: keepalive`) is not used since it cannot carry cursor state.', + beats.length === 0 ? 'FAILURE' : 'WARNING', + { + errorMessage: `The stream carried ${session.sseComments.length} SSE comment line(s) (e.g. ${JSON.stringify(session.sseComments[0])}), which cannot carry cursor state.` + } + ) + ); + + return out; + } + + /** The `StreamEventsResult`, which only a server-initiated close produces. */ + /** + * Grade the three rows that need the *server* to end the stream: + * `stream-terminated-ends-subscription` and both `stream-final-result-*`. + * + * These cannot be graded on the main session. The harness cancels that one + * to test `stream-cancel-stops-delivery`, and on Streamable HTTP a + * client-side abort is terminal, so no final frame is ever sent. A + * server-ended stream is a different stream. + * + * Termination is one-shot for the event type and the events scenarios share + * a fixture process, so this deliberately refuses to terminate the type the + * rest of the run depends on. A fixture with only one push-capable type gets + * untestable rather than a poisoned suite. + */ + private async terminateChecks( + ctx: RunContext, + descriptors: EventDescriptor[], + usedName: string, + hasTerminate: boolean + ): Promise { + // Severity follows each row's own wording, not the reason they share. + // Terminating is a MUST. The final-result shape is a shape the SEP states + // without a keyword, and the timing sentence has none either, so when + // neither can be observed they report untestable at WARNING; flattening + // all three to FAILURE overstates two of them. A shape that is observed + // and wrong still fails, per the header's shape-row rule. + const untestableHere = (reason: string): ConformanceCheck[] => [ + ...untestableAll( + ['sep-3415-stream-terminated-ends-subscription'], + reason + ), + ...untestableAll( + [ + 'sep-3415-stream-final-result-shape', + 'sep-3415-stream-final-result-timing', + RESULT_TYPE_ID + ], + reason, + 'WARNING' + ) + ]; + + if (!hasTerminate) { + return untestableHere( + `The subscription was not terminated during the run, and the harness cannot revoke one over the protocol. Needs a fixture exposing the \`${EVENTS_CONTROL_TERMINATE}\` control.` + ); + } + + const spare = descriptors + .filter((d) => deliveryModes(d).includes('push')) + .map((d) => descriptorName(d)) + .find((n): n is string => !!n && n !== usedName); + if (!spare) { + return untestableHere( + `Terminating an event type is one-shot for the life of the fixture, and \`${usedName}\` is the only push-capable type on offer, so terminating it would break every scenario after this one. Needs a second push-capable type the run does not otherwise depend on.` + ); + } + + const args = minimalArguments( + descriptors.find((d) => descriptorName(d) === spare)! + ); + if (args === undefined) { + return untestableHere( + `Event type \`${spare}\` declares required \`inputSchema\` properties the harness cannot satisfy, so no stream could be opened to terminate.` + ); + } + + const session = await openEventStream( + ctx.serverUrl, + ctx.specVersion, + { name: spare, arguments: args, cursor: null }, + { openTimeoutMs: ACTIVE_MS } + ); + try { + if (!session.contentType?.includes('text/event-stream')) { + return untestableHere( + `\`${EVENTS_STREAM_METHOD}\` did not open a stream for \`${spare}\`, so there was nothing to terminate.` + ); + } + await session.waitFor( + (n) => n.method === EVENTS_ACTIVE_NOTIFICATION, + ACTIVE_MS + ); + + const control = await ctx.connect(); + try { + await fireControl(control, EVENTS_CONTROL_TERMINATE, spare); + } finally { + await control.close(); + } + + const terminated = await session.waitFor( + (n) => n.method === EVENTS_TERMINATED_NOTIFICATION, + TERMINATE_MS + ); + await session.settle(500); + + const out: ConformanceCheck[] = []; + out.push( + terminated + ? eventsCheck( + 'sep-3415-stream-terminated-ends-subscription', + 'Only `notifications/events/terminated` ends the subscription.', + session.open ? 'FAILURE' : 'SUCCESS', + { + errorMessage: session.open + ? `\`${EVENTS_TERMINATED_NOTIFICATION}\` arrived for \`${spare}\` but the stream stayed open.` + : undefined, + details: { name: spare } + } + ) + : eventsCheck( + 'sep-3415-stream-terminated-ends-subscription', + 'Only `notifications/events/terminated` ends the subscription.', + 'FAILURE', + { + errorMessage: `The \`${EVENTS_CONTROL_TERMINATE}\` control was called for \`${spare}\` but no \`${EVENTS_TERMINATED_NOTIFICATION}\` arrived within ${TERMINATE_MS}ms.` + } + ) + ); + out.push(...this.finalResultChecks(session)); + return out; + } finally { + await session.cancel(); + } + } + + private finalResultChecks( + session: StreamSession, + emitUntestable = true + ): ConformanceCheck[] { + const result = session.finalResult?.result; + if (result === undefined) { + // The main session calls this opportunistically: a server that closed + // the stream itself has answered these rows, and one that did not leaves + // them to terminateChecks, which knows why they could not be exercised. + if (!emitUntestable) return []; + return untestableAll( + [ + 'sep-3415-stream-final-result-shape', + 'sep-3415-stream-final-result-timing', + RESULT_TYPE_ID + ], + 'The server did not close the stream itself, so no final frame was sent.', + 'WARNING' + ); + } + const out: ConformanceCheck[] = []; + // `_meta` and `resultType` are base-protocol fields on the common `Result` + // interface, not information this extension's final frame carries. Servers + // MUST include `resultType`, so counting it as a payload field fails every + // conformant server; mcpkit is how that surfaced. + const keys = isObject(result) + ? Object.keys(result).filter((k) => k !== '_meta' && k !== 'resultType') + : ['']; + const meta = + isObject(result) && isObject(result._meta) ? result._meta : undefined; + const subscriptionId = meta?.[SUBSCRIPTION_ID_META]; + const shapeDesc = + 'The `StreamEventsResult` is an empty result that carries only `resultType` and the subscription ID in `_meta`, mirroring `SubscriptionsListenResult`. It carries no information.'; + out.push( + keys.length > 0 + ? eventsCheck( + 'sep-3415-stream-final-result-shape', + shapeDesc, + 'FAILURE', + { + errorMessage: `The final result carried ${keys.join(', ')}; it is defined to carry only \`resultType\` and the subscription ID in \`_meta\`.`, + details: { result } + } + ) + : subscriptionId === undefined || subscriptionId === null + ? eventsCheck( + 'sep-3415-stream-final-result-shape', + shapeDesc, + 'FAILURE', + { + errorMessage: `The final result's \`_meta\` carried no \`${SUBSCRIPTION_ID_META}\`, which SEP-3415 puts there.`, + details: { result } + } + ) + : eventsCheck( + 'sep-3415-stream-final-result-shape', + shapeDesc, + 'SUCCESS', + { + details: { result } + } + ) + ); + out.push(resultTypeCheck([{ method: EVENTS_STREAM_METHOD, result }])); + out.push( + eventsCheck( + 'sep-3415-stream-final-result-timing', + 'The final result is sent whenever the server can write a final frame; on Streamable HTTP only when the server initiates the close.', + 'SUCCESS', + { + details: { + note: 'The server initiated the close and wrote a final frame.' + } + } + ) + ); + return out; + } + + /** Concurrent streams, which a request-concurrency cap would strangle. */ + private async concurrencyChecks( + ctx: RunContext, + name: string, + args: Record + ): Promise { + const description = + 'Server SDKs MUST exempt `events/stream` from any general request-concurrency cap, since each push subscription is a long-lived request that never completes until cancelled.'; + const sessions: StreamSession[] = []; + try { + for (let i = 0; i < 3; i++) { + sessions.push( + await openEventStream( + ctx.serverUrl, + ctx.specVersion, + { name, arguments: args, cursor: null }, + { openTimeoutMs: ACTIVE_MS } + ) + ); + } + const streamed = sessions.filter((s) => + s.contentType?.includes('text/event-stream') + ); + const confirmed = await Promise.all( + streamed.map((s) => + s.waitFor((n) => n.method === EVENTS_ACTIVE_NOTIFICATION, ACTIVE_MS) + ) + ); + const live = confirmed.filter(Boolean).length; + return [ + live === sessions.length + ? eventsCheck( + 'sep-3415-stream-exempt-from-concurrency-cap', + description, + 'SUCCESS', + { details: { concurrent: live } } + ) + : eventsCheck( + 'sep-3415-stream-exempt-from-concurrency-cap', + description, + 'FAILURE', + { + errorMessage: `Opened ${sessions.length} concurrent \`${EVENTS_STREAM_METHOD}\` requests; only ${live} confirmed with \`${EVENTS_ACTIVE_NOTIFICATION}\`.`, + details: { + statuses: sessions.map((s) => ({ + status: s.status, + contentType: s.contentType, + error: s.error + })) + } + } + ), + this.resourceExhaustedCheck(sessions.map((s) => s.error)) + ]; + } finally { + await Promise.all(sessions.map((s) => s.cancel())); + } + } + + /** + * What `events_conformance_quota` reported, or undefined when the server has + * no such control, which is the normal case. + */ + private async readQuota(conn: Connection): Promise { + if (!(await hasControl(conn, EVENTS_CONTROL_QUOTA))) return undefined; + const text = await askControl(conn, EVENTS_CONTROL_QUOTA, {}); + let parsed: unknown; + try { + parsed = text === undefined ? undefined : JSON.parse(text); + } catch { + parsed = undefined; + } + if ( + isObject(parsed) && + typeof parsed.name === 'string' && + typeof parsed.max === 'number' && + Number.isInteger(parsed.max) && + parsed.max >= 0 + ) { + return { name: parsed.name, max: parsed.max }; + } + return { malformed: text ?? '(no text, or the call failed)' }; + } + + /** + * `-32025`, provoked on the type the server says is capped. + * + * Opens streams one at a time, up to one past the reported cap, and stops at + * the first refusal. Refused earlier than the cap still counts, since other + * scenarios in the run may hold subscriptions under the same principal. + */ + private async quotaCheck( + ctx: RunContext, + quota: QuotaReport, + descriptors: EventDescriptor[] + ): Promise { + const id = 'sep-3415-error-resource-exhausted'; + const description = + '`-32025 ResourceExhausted` — a server-imposed limit or quota was reached. `data.limit` names it (e.g. `"subscriptions"`).'; + const untestable = (reason: string) => + untestableCheck(id, id, description, reason, [EVENTS_SPEC_REF]); + + if ('malformed' in quota) { + return untestable( + `\`${EVENTS_CONTROL_QUOTA}\` answered ${quota.malformed}, where \`{"name": "", "max": }\` was expected, so no capped type was known to probe.` + ); + } + const target = descriptors.find((d) => descriptorName(d) === quota.name); + const args = target ? minimalArguments(target) : undefined; + if (!target || args === undefined) { + return untestable( + target + ? `\`${quota.name}\`, the type \`${EVENTS_CONTROL_QUOTA}\` reported as capped, declares required \`inputSchema\` properties the harness cannot satisfy.` + : `\`${EVENTS_CONTROL_QUOTA}\` reported \`${quota.name}\` as capped, but \`events/list\` does not serve it.` + ); + } + + const sessions: StreamSession[] = []; + try { + let refusal: StreamSession['error']; + for (let i = 0; i <= quota.max && !refusal; i++) { + const s = await openEventStream( + ctx.serverUrl, + ctx.specVersion, + { name: quota.name, arguments: args, cursor: null }, + { openTimeoutMs: ACTIVE_MS } + ); + sessions.push(s); + refusal = s.error; + } + if (!refusal) { + return untestable( + `\`${EVENTS_CONTROL_QUOTA}\` reported a cap of ${quota.max} on \`${quota.name}\`, and ${sessions.length} streams opened on it without a refusal, so the cap it reported was never reached.` + ); + } + if (refusal.code !== EVENTS_RESOURCE_EXHAUSTED) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `Stream ${sessions.length} on \`${quota.name}\`, past its reported cap of ${quota.max}, was refused with ${refusal.code} ${refusal.message}. A refusal for a quota is \`${EVENTS_RESOURCE_EXHAUSTED}\`.`, + details: { code: refusal.code, message: refusal.message } + }); + } + return this.resourceExhaustedCheck([refusal]); + } finally { + await Promise.all(sessions.map((s) => s.cancel())); + } + } + + /** + * The `-32025` row from the error table. + * + * A conformant server reaches no limit here, so this usually reports + * untestable. It grades when one refuses, which is worth claiming rather than + * leaving to the concurrency row: that row says streams are exempt from the + * cap, this one says a refusal for a quota names the quota. + */ + private resourceExhaustedCheck( + errors: (StreamSession['error'] | undefined)[] + ): ConformanceCheck { + const id = 'sep-3415-error-resource-exhausted'; + const description = + '`-32025 ResourceExhausted` — a server-imposed limit or quota was reached. `data.limit` names it (e.g. `"subscriptions"`).'; + const hit = errors.find((e) => e?.code === EVENTS_RESOURCE_EXHAUSTED); + if (!hit) { + return untestableCheck( + id, + id, + description, + `No request in this run was refused for a server-imposed limit, so the code was never provoked. A server that caps concurrent subscriptions answers it for the third stream above, or can name a capped event type through the \`${EVENTS_CONTROL_QUOTA}\` control so the cap is probed on its own.`, + [EVENTS_SPEC_REF] + ); + } + const limit = isObject(hit.data) ? hit.data.limit : undefined; + return typeof limit === 'string' && limit.length > 0 + ? eventsCheck(id, description, 'SUCCESS', { + details: { limit, message: hit.message } + }) + : eventsCheck(id, description, 'WARNING', { + errorMessage: `\`${EVENTS_RESOURCE_EXHAUSTED}\` carried \`data.limit\` ${describeValue(limit)}. Without it a client cannot tell which quota it hit, so it cannot know what to stop doing.`, + details: { data: hit.data, message: hit.message } + }); + } + + /** An invalid subscription answers an error and opens no stream. */ + private async errorBeforeOpenChecks( + ctx: RunContext, + args: Record + ): Promise { + const unknown = `conformance.nonexistent.${Date.now()}`; + const session = await openEventStream( + ctx.serverUrl, + ctx.specVersion, + { name: unknown, arguments: args, cursor: null }, + { openTimeoutMs: 1000 } + ); + try { + const description = + 'If the subscription is invalid (`NotFound`, `Forbidden`, `InvalidParams`, `Unsupported`), the server responds immediately with a JSON-RPC error and no stream is opened.'; + if (!session.error) { + return [ + eventsCheck( + 'sep-3415-stream-error-before-open', + description, + 'FAILURE', + { + errorMessage: `\`${EVENTS_STREAM_METHOD}\` for unknown event type \`${unknown}\` did not answer a JSON-RPC error (HTTP ${session.status}, content-type ${describeValue(session.contentType)}).`, + details: { + notifications: session.notifications.map((n) => n.method) + } + } + ) + ]; + } + return [ + session.error.code === EVENTS_NOT_FOUND + ? eventsCheck( + 'sep-3415-stream-error-before-open', + description, + 'SUCCESS', + { details: { code: session.error.code } } + ) + : eventsCheck( + 'sep-3415-stream-error-before-open', + description, + 'WARNING', + { + errorMessage: `Unknown event type answered ${session.error.code} ${session.error.message}; the document names \`-32023 NotFound\` for a referenced entity that does not exist.`, + details: { error: session.error } + } + ) + ]; + } finally { + await session.cancel(); + } + } + + /** + * Remove, retype and extend a minted type under a live stream, one type per + * probe, and grade what the stream sees. + */ + private async catalogChecks(ctx: RunContext): Promise { + const conn = await ctx.connect(); + try { + if (!(await hasControl(conn, EVENTS_CONTROL_CATALOG))) { + return Object.values(CATALOG_ROWS).map((row) => + catalogUntestable( + row, + `A healthy server never changes its catalog mid-run. Grading this needs the \`${EVENTS_CONTROL_CATALOG}\` control, which adds a type and then removes, retypes or extends it.` + ) + ); + } + const out: ConformanceCheck[] = []; + + // Removal: the termination signal, and the NotFound it carries. + const removed = await this.watchCatalogOp(ctx, conn, 'remove'); + if ('untestable' in removed) { + out.push( + catalogUntestable(CATALOG_ROWS.terminates, removed.untestable), + catalogUntestable(CATALOG_ROWS.notFound, removed.untestable) + ); + } else { + out.push( + removed.terminated + ? eventsCheck( + CATALOG_ROWS.terminates.id, + CATALOG_ROWS.terminates.description, + 'SUCCESS', + { + details: { + removed: removed.name, + note: 'Push leg only; the minted type does not offer webhook.' + } + } + ) + : eventsCheck( + CATALOG_ROWS.terminates.id, + CATALOG_ROWS.terminates.description, + 'WARNING', + { + errorMessage: `Removed \`${removed.name}\` while a stream was open on it, and no \`${EVENTS_TERMINATED_NOTIFICATION}\` arrived within ${TERMINATE_MS}ms. The subscriber is left waiting on a name that is gone.` + } + ) + ); + const err = terminatedError(removed.terminated); + out.push( + !removed.terminated + ? catalogUntestable( + CATALOG_ROWS.notFound, + 'The removal ended no stream, so there was no termination error to read.' + ) + : err?.code === EVENTS_NOT_FOUND && err.data?.kind === 'event' + ? eventsCheck( + CATALOG_ROWS.notFound.id, + CATALOG_ROWS.notFound.description, + 'SUCCESS', + { details: { error: removed.terminated.params.error } } + ) + : eventsCheck( + CATALOG_ROWS.notFound.id, + CATALOG_ROWS.notFound.description, + 'FAILURE', + { + errorMessage: `The stream on removed type \`${removed.name}\` ended with ${describeValue(removed.terminated.params.error)}, expected ${EVENTS_NOT_FOUND} NotFound with \`data.kind: "event"\`, which tells a client the type is gone rather than its subscription.` + } + ) + ); + } + + // A breaking in-place change: Unsupported with schema_changed. + const retyped = await this.watchCatalogOp(ctx, conn, 'retype'); + if ('untestable' in retyped) { + out.push( + catalogUntestable(CATALOG_ROWS.schemaChanged, retyped.untestable) + ); + } else if (!retyped.terminated) { + out.push( + catalogUntestable( + CATALOG_ROWS.schemaChanged, + `Retyping \`${retyped.name}\` in place ended no stream within ${TERMINATE_MS}ms, so there was no error to read.` + ) + ); + } else { + const err = terminatedError(retyped.terminated); + const ok = + err?.code === EVENTS_UNSUPPORTED && + err.data?.reason === 'schema_changed' && + (err.data?.feature === 'inputSchema' || + err.data?.feature === 'payloadSchema'); + out.push( + ok + ? eventsCheck( + CATALOG_ROWS.schemaChanged.id, + CATALOG_ROWS.schemaChanged.description, + 'SUCCESS', + { details: { error: retyped.terminated.params.error } } + ) + : eventsCheck( + CATALOG_ROWS.schemaChanged.id, + CATALOG_ROWS.schemaChanged.description, + 'FAILURE', + { + errorMessage: `The stream on retyped \`${retyped.name}\` ended with ${describeValue(retyped.terminated.params.error)}, expected ${EVENTS_UNSUPPORTED} Unsupported with \`data: {feature, reason: "schema_changed"}\`. Not Forbidden: the principal's access did not change.` + } + ) + ); + } + + // An additive change: the stream stays open. + const extended = await this.watchCatalogOp(ctx, conn, 'extend'); + out.push( + 'untestable' in extended + ? catalogUntestable(CATALOG_ROWS.additive, extended.untestable) + : extended.terminated + ? eventsCheck( + CATALOG_ROWS.additive.id, + CATALOG_ROWS.additive.description, + 'FAILURE', + { + errorMessage: `Extending \`${extended.name}\` with an optional property ended its stream (${describeValue(extended.terminated.params.error)}). An additive change MUST NOT terminate subscriptions.` + } + ) + : eventsCheck( + CATALOG_ROWS.additive.id, + CATALOG_ROWS.additive.description, + 'SUCCESS', + { + details: { + extended: extended.name, + watchedMs: ADDITIVE_WATCH_MS + } + } + ) + ); + return out; + } finally { + await conn.close(); + } + } + + /** + * Mint a type, open a stream on it, apply op, and report the + * `notifications/events/terminated` it caused, if any. Cleans up the type + * unless op removed it. + */ + private async watchCatalogOp( + ctx: RunContext, + conn: Connection, + op: 'remove' | 'retype' | 'extend' + ): Promise< + { untestable: string } | { name: string; terminated?: StreamNotification } + > { + const added = await askControl(conn, EVENTS_CONTROL_CATALOG, { op: 'add' }); + const name = added?.match(/^ok:\s*(\S+)/)?.[1]; + if (!name) { + return { + untestable: `\`${EVENTS_CONTROL_CATALOG}\` {op: "add"} did not add a type (answered ${JSON.stringify(added)}).` + }; + } + const session = await openEventStream( + ctx.serverUrl, + ctx.specVersion, + { name, arguments: {}, cursor: null }, + { openTimeoutMs: ACTIVE_MS } + ); + try { + if (!session.contentType?.includes('text/event-stream')) { + return { + untestable: `\`${EVENTS_STREAM_METHOD}\` did not open a stream on minted type \`${name}\`, so there was nothing for the ${op} to act on.` + }; + } + await session.waitFor( + (n) => n.method === EVENTS_ACTIVE_NOTIFICATION, + ACTIVE_MS + ); + const answered = await askControl(conn, EVENTS_CONTROL_CATALOG, { + op, + name + }); + if (!answered?.startsWith('ok')) { + return { + untestable: `\`${EVENTS_CONTROL_CATALOG}\` {op: "${op}"} was refused for \`${name}\` (answered ${JSON.stringify(answered)}).` + }; + } + const terminated = await session.waitFor( + (n) => n.method === EVENTS_TERMINATED_NOTIFICATION, + op === 'extend' ? ADDITIVE_WATCH_MS : TERMINATE_MS + ); + return { name, terminated }; + } finally { + await session.cancel(); + if (op !== 'remove') { + await askControl(conn, EVENTS_CONTROL_CATALOG, { op: 'remove', name }); + } + } + } + + /** Open a stream on an event type whose `delivery` omits `push`. */ + private async unsupportedModeCheck( + ctx: RunContext, + descriptors: EventDescriptor[] + ): Promise { + const id = 'sep-3415-stream-mode-unsupported'; + const description = + 'A stream against an event type whose `delivery` does not list `push` returns `-32026 Unsupported`, and opens nothing.'; + const nonPush = descriptors.find( + (d) => + descriptorName(d) !== undefined && + !deliveryModes(d).includes('push') && + minimalArguments(d) !== undefined + ); + if (!nonPush) { + return untestableCheck( + id, + id, + description, + 'Every event type the server offers advertises `push` delivery, so there is no event type to probe the unsupported-mode path with.', + [EVENTS_SPEC_REF] + ); + } + const name = descriptorName(nonPush)!; + const session = await openEventStream( + ctx.serverUrl, + ctx.specVersion, + { name, arguments: minimalArguments(nonPush)!, cursor: null }, + { openTimeoutMs: 1000 } + ); + try { + if (!session.error) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `Event type \`${name}\` does not advertise \`push\` delivery (\`delivery\` is ${JSON.stringify(nonPush.delivery)}) but \`${EVENTS_STREAM_METHOD}\` opened a stream instead of answering an error.`, + details: { + name, + delivery: deliveryModes(nonPush), + notifications: session.notifications.map((n) => n.method) + } + }); + } + return session.error.code === EVENTS_UNSUPPORTED + ? eventsCheck(id, description, 'SUCCESS', { + details: { + name, + code: session.error.code, + data: session.error.data + } + }) + : eventsCheck(id, description, 'FAILURE', { + errorMessage: `Streaming \`${name}\`, which does not offer push delivery, answered ${session.error.code} ${session.error.message}, expected ${EVENTS_UNSUPPORTED} Unsupported.`, + details: { + name, + delivery: deliveryModes(nonPush), + error: session.error + } + }); + } finally { + await session.cancel(); + } + } +} + +/** Keep the first check emitted per id, so a fallback path cannot double-report. */ +/** What `events_conformance_quota` answered: a capped type, or garbage. */ +type QuotaReport = { name: string; max: number } | { malformed: string }; + +function dedupe(checks: ConformanceCheck[]): ConformanceCheck[] { + const seen = new Set(); + return checks.filter((c) => { + if (seen.has(c.id)) return false; + seen.add(c.id); + return true; + }); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/src/scenarios/server/events/receiver.ts b/src/scenarios/server/events/receiver.ts new file mode 100644 index 00000000..740be405 --- /dev/null +++ b/src/scenarios/server/events/receiver.ts @@ -0,0 +1,272 @@ +/** + * A callback endpoint for the webhook-delivery scenario. + * + * The scenario needs to be the receiver, not just the subscriber: the + * signature, the Standard Webhooks headers, the verification challenge and the + * retry behaviour are all only observable from the endpoint the server POSTs + * to. So this runs a real HTTP server and records every request verbatim, + * including the raw body bytes, because the signature is computed over those + * and re-serializing the JSON would change them. + * + * Per-path behaviour lets one receiver serve every probe: a path that echoes + * the challenge and accepts, one that redirects, one that refuses permanently, + * and one that fails a few times before accepting. + * + * The receiver can also publish `/.well-known/mcp-webhook-receiver.json`, which + * is the fourth way the document lets a server confirm intent: an origin that + * serves it has declared consent for the path prefixes it names, and no + * challenge POST is needed. `publishWellKnown` turns it on and `wellKnownFetches` + * counts the GETs, which is how the scenario tells that path apart from a server + * that simply skipped verification. + * + * Every path answers the verification challenge first, whatever its behaviour, + * because a probe exists to misbehave on deliveries. A 410 probe that also + * refused the challenge would never get subscribed by a server that verifies + * inside events/subscribe, and its row would go untestable for a reason that + * has nothing to do with 410. `wrong-challenge` is the one path that fails the + * handshake, since that is its whole job. + */ + +import http from 'node:http'; +import net from 'node:net'; +import type { AddressInfo } from 'node:net'; + +export interface ReceivedDelivery { + path: string; + method: string; + headers: Record; + /** The body exactly as it arrived, which is what the signature covers. */ + rawBody: string; + /** Parsed body, when it was JSON. */ + json?: Record; + /** Milliseconds since the receiver started. */ + atMs: number; + /** What this receiver answered. */ + respondedStatus: number; +} + +export type PathBehaviour = + | { kind: 'accept' } + | { kind: 'redirect'; to: string } + | { kind: 'gone' } + | { kind: 'too-large' } + | { kind: 'fail-then-accept'; failures: number; status: number } + | { kind: 'wrong-challenge' }; + +/** + * What the `wrong-challenge` path echoes instead of the nonce. + * + * Distinctive on purpose: the scenario looks for this string in whatever error + * the server reports afterwards, because the document says a failed handshake + * surfaces as a category and never as the endpoint's own response body. + */ +export const WRONG_CHALLENGE_ECHO = 'not-the-nonce'; + +/** Where a receiver declares which of its paths accept MCP deliveries. */ +export const RECEIVER_WELL_KNOWN_PATH = + '/.well-known/mcp-webhook-receiver.json'; + +export interface Receiver { + /** Base URL of the receiver, e.g. `http://127.0.0.1:53211`. */ + readonly url: string; + readonly deliveries: ReceivedDelivery[]; + /** Deliveries on one path, in arrival order. */ + on(path: string): ReceivedDelivery[]; + /** Set how a path answers. Unknown paths accept. */ + behave(path: string, behaviour: PathBehaviour): void; + /** + * Serve the well-known document, declaring `prefixes` as consenting paths. + * Until this is called the path answers 404, which is what a receiver that + * cannot publish same-origin content looks like. + */ + publishWellKnown(prefixes: string[]): void; + /** How many times the well-known document has been fetched. */ + wellKnownFetches(): number; + /** Resolve once a delivery on `path` matches, or undefined at the deadline. */ + waitFor( + path: string, + predicate: (d: ReceivedDelivery) => boolean, + timeoutMs: number + ): Promise; + close(): Promise; +} + +export async function startReceiver(host = '127.0.0.1'): Promise { + const deliveries: ReceivedDelivery[] = []; + const behaviours = new Map(); + const failureCounts = new Map(); + const startedAt = Date.now(); + let wellKnown: string[] | undefined; + let wellKnownGets = 0; + + const server = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on('data', (c: Buffer) => chunks.push(c)); + req.on('end', () => { + const rawBody = Buffer.concat(chunks).toString('utf8'); + const path = (req.url ?? '/').split('?')[0]; + let json: Record | undefined; + try { + const parsed: unknown = JSON.parse(rawBody); + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + json = parsed as Record; + } + } catch { + // Not JSON, which is itself something the scenario grades. + } + + // The well-known document is not a delivery, so it is answered before the + // per-path behaviours and recorded only as a fetch count. + if (path === RECEIVER_WELL_KNOWN_PATH) { + if (!wellKnown) { + res.writeHead(404, { 'content-type': 'application/json' }); + res.end('{}'); + return; + } + wellKnownGets += 1; + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ receivers: wellKnown })); + return; + } + + const behaviour = behaviours.get(path) ?? { kind: 'accept' }; + const respond = (status: number, body?: string): void => { + deliveries.push({ + path, + method: req.method ?? 'GET', + headers: Object.fromEntries( + Object.entries(req.headers).map(([k, v]) => [ + k.toLowerCase(), + Array.isArray(v) ? v.join(', ') : (v ?? '') + ]) + ), + rawBody, + json, + atMs: Date.now() - startedAt, + respondedStatus: status + }); + res.writeHead(status, { 'content-type': 'application/json' }); + res.end(body ?? '{}'); + }; + + const challenge = json?.challenge; + const isChallenge = + json?.type === 'verification' && typeof challenge === 'string'; + if (isChallenge && behaviour.kind !== 'wrong-challenge') { + respond(200, JSON.stringify({ challenge })); + return; + } + + switch (behaviour.kind) { + case 'redirect': + res.writeHead(302, { location: behaviour.to }); + deliveries.push({ + path, + method: req.method ?? 'GET', + headers: {}, + rawBody, + json, + atMs: Date.now() - startedAt, + respondedStatus: 302 + }); + res.end(); + return; + case 'gone': + respond(410); + return; + case 'too-large': + respond(413); + return; + case 'wrong-challenge': + respond(200, JSON.stringify({ challenge: WRONG_CHALLENGE_ECHO })); + return; + case 'fail-then-accept': { + const seen = failureCounts.get(path) ?? 0; + if (seen < behaviour.failures) { + failureCounts.set(path, seen + 1); + respond(behaviour.status); + return; + } + break; + } + default: + break; + } + + // A challenge without the `type` discriminator: still echo it, so a + // server that got the envelope shape slightly wrong is graded on the + // row about the shape rather than locked out of every other row. + if (typeof challenge === 'string') { + respond(200, JSON.stringify({ challenge })); + return; + } + respond(200); + }); + }); + + await new Promise((resolve) => server.listen(0, host, resolve)); + const { port } = server.address() as AddressInfo; + + return { + url: `http://${host}:${port}`, + deliveries, + on(path) { + return deliveries.filter((d) => d.path === path); + }, + behave(path, behaviour) { + behaviours.set(path, behaviour); + }, + publishWellKnown(prefixes) { + wellKnown = prefixes; + }, + wellKnownFetches() { + return wellKnownGets; + }, + async waitFor(path, predicate, timeoutMs) { + const deadline = Date.now() + timeoutMs; + for (;;) { + const hit = deliveries.filter((d) => d.path === path).find(predicate); + if (hit) return hit; + if (Date.now() >= deadline) return undefined; + await new Promise((resolve) => setTimeout(resolve, 100)); + } + }, + async close() { + await new Promise((resolve) => server.close(() => resolve())); + } + }; +} + +/** + * A bare TCP listener that records whether anything connected to it, and + * nothing else. + * + * The SSRF probe points an `https://127.0.0.1` callback here. The scheme is + * valid, so the only rule that can refuse it is routability, and a refusal is + * only evidence of that rule if the server never dialled: a server that skips + * the check and then fails its own verification POST answers with an error too, + * which reads as a refusal from the subscribe alone. The listener speaks no + * TLS and answers nothing, so every connection is dropped as soon as it lands. + */ +export interface Canary { + readonly port: number; + connections(): number; + close(): Promise; +} + +export async function startCanary(): Promise { + let count = 0; + const server = net.createServer((socket) => { + count++; + socket.destroy(); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const { port } = server.address() as AddressInfo; + return { + port, + connections: () => count, + async close() { + await new Promise((resolve) => server.close(() => resolve())); + } + }; +} diff --git a/src/scenarios/server/events/stream.ts b/src/scenarios/server/events/stream.ts new file mode 100644 index 00000000..2a240938 --- /dev/null +++ b/src/scenarios/server/events/stream.ts @@ -0,0 +1,246 @@ +/** + * A long-lived `events/stream` request, which the `Connection` abstraction + * cannot express. + * + * `conn.request()` resolves when the response for its JSON-RPC id arrives, and + * that is exactly what a push stream withholds: the `StreamEventsResult` is + * the last frame before close, so a scenario that awaited it would block for + * the life of the subscription and see none of the notifications it is meant + * to grade. `readSseJsonRpcResponse` has the same shape of problem — it stops + * reading at the first frame matching the request id. + * + * So this opens the POST itself, hands back a session while the stream is + * still open, and appends frames as they arrive. The scenario waits for the + * frames it needs, grades them, then aborts. Aborting is also what the + * document calls the client-side cancel on Streamable HTTP, so the cancel + * checks come out of the same mechanism rather than a second code path. + * + * stdio is out of scope here: the runner addresses a URL, and the document's + * stdio rules (`notifications/cancelled`, the MAY on a final result) have no + * HTTP analogue. Rows that only bite on stdio are reported untestable by the + * scenario rather than graded against a transport the harness cannot open. + */ + +import { + buildStandardHeaders, + withRequestMeta, + type JsonRpcResponse +} from '../../../connection'; +import type { SpecVersion } from '../../../types'; +import { isObject } from './helpers'; + +/** A JSON-RPC notification as it arrived on the stream. */ +export interface StreamNotification { + method: string; + params: Record; + /** Milliseconds since the stream was opened, for the heartbeat interval. */ + atMs: number; +} + +export interface StreamSession { + /** The JSON-RPC id of the `events/stream` request, echoed in `_meta`. */ + readonly requestId: number; + /** HTTP status of the POST that opened the stream. */ + readonly status: number; + readonly contentType?: string; + /** Set when the server answered with an immediate JSON-RPC error. */ + readonly error?: { code: number; message: string; data?: unknown }; + /** Set when a response for `requestId` arrived (the `StreamEventsResult`). */ + readonly finalResult?: JsonRpcResponse; + /** Notifications in arrival order. */ + readonly notifications: StreamNotification[]; + /** Frames that were neither a notification nor this request's response. */ + readonly foreignFrames: unknown[]; + /** SSE comment lines (`: keepalive`), which the document rules out. */ + readonly sseComments: string[]; + /** Whether the reader is still running. */ + readonly open: boolean; + /** Resolve once a notification matches, or undefined at the deadline. */ + waitFor( + predicate: (n: StreamNotification) => boolean, + timeoutMs: number + ): Promise; + /** Resolve after `ms`, or earlier if the stream closes. */ + settle(ms: number): Promise; + /** Abort the request stream, which is the client-side cancel on HTTP. */ + cancel(): Promise; +} + +/** + * Open `events/stream` and return once the server has answered the POST. + * + * `options.method` opens another long-lived request with the same shape + * instead, which is how the discovery scenario holds a `subscriptions/listen` + * stream open: confirmation first, notifications after, a result only on + * close. + */ +export async function openEventStream( + serverUrl: string, + specVersion: SpecVersion, + params: Record, + options: { openTimeoutMs?: number; method?: string } = {} +): Promise { + const method = options.method ?? 'events/stream'; + const requestId = Math.floor(Math.random() * 1_000_000) + 1; + const headers = buildStandardHeaders(method, params, { + specVersion + }); + const body = JSON.stringify({ + jsonrpc: '2.0', + id: requestId, + method, + params: withRequestMeta(params, specVersion) + }); + + const controller = new AbortController(); + const openedAt = Date.now(); + const notifications: StreamNotification[] = []; + const foreignFrames: unknown[] = []; + const sseComments: string[] = []; + const state = { + open: true, + error: undefined as StreamSession['error'], + finalResult: undefined as JsonRpcResponse | undefined + }; + + const res = await fetch(serverUrl, { + method: 'POST', + headers, + body, + signal: controller.signal + }); + const contentType = res.headers.get('content-type') ?? undefined; + + const ingest = (frame: unknown): void => { + if (!isObject(frame)) { + foreignFrames.push(frame); + return; + } + if (typeof frame.method === 'string' && frame.id === undefined) { + notifications.push({ + method: frame.method, + params: isObject(frame.params) ? frame.params : {}, + atMs: Date.now() - openedAt + }); + return; + } + if (frame.id === requestId) { + state.finalResult = frame as unknown as JsonRpcResponse; + if (isObject(frame.error)) { + state.error = { + code: Number(frame.error.code), + message: String(frame.error.message ?? ''), + data: frame.error.data + }; + } + return; + } + foreignFrames.push(frame); + }; + + // A JSON body means the server answered rather than streamed: either the + // immediate error the document requires for an invalid subscription, or a + // server that does not implement push at all. + if (!contentType?.includes('text/event-stream')) { + state.open = false; + const text = await res.text(); + try { + ingest(JSON.parse(text)); + } catch { + foreignFrames.push(text); + } + } else { + void readFrames(res, ingest, sseComments).finally(() => { + state.open = false; + }); + } + + const session: StreamSession = { + requestId, + status: res.status, + contentType, + get error() { + return state.error; + }, + get finalResult() { + return state.finalResult; + }, + get open() { + return state.open; + }, + notifications, + foreignFrames, + sseComments, + async waitFor(predicate, timeoutMs) { + const deadline = Date.now() + timeoutMs; + for (;;) { + const hit = notifications.find(predicate); + if (hit) return hit; + if (!state.open || Date.now() >= deadline) { + return notifications.find(predicate); + } + await sleep(Math.min(100, Math.max(1, deadline - Date.now()))); + } + }, + async settle(ms) { + const deadline = Date.now() + ms; + while (state.open && Date.now() < deadline) { + await sleep(Math.min(100, Math.max(1, deadline - Date.now()))); + } + }, + async cancel() { + controller.abort(); + state.open = false; + } + }; + + // Give a streaming server a moment to write its confirmation frame, so a + // caller that opens and immediately grades is not racing the first write. + if (state.open) { + await session.waitFor(() => true, options.openTimeoutMs ?? 2000); + } + return session; +} + +/** Read SSE `data:` frames until the stream ends or the request is aborted. */ +async function readFrames( + res: Response, + ingest: (frame: unknown) => void, + sseComments: string[] +): Promise { + if (!res.body) return; + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + try { + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + if (!value) continue; + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split(/\r?\n/); + buffer = lines.pop() ?? ''; + for (const line of lines) { + if (line.startsWith(':')) { + sseComments.push(line); + continue; + } + if (!line.startsWith('data:')) continue; + const payload = line.slice(5).trim(); + if (!payload) continue; + try { + ingest(JSON.parse(payload)); + } catch { + ingest(payload); + } + } + } + } catch { + // Aborted by cancel(), or the connection dropped. Either way the frames + // that arrived are what the scenario grades. + } +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/src/scenarios/server/events/webhook-delivery.ts b/src/scenarios/server/events/webhook-delivery.ts new file mode 100644 index 00000000..ce828cad --- /dev/null +++ b/src/scenarios/server/events/webhook-delivery.ts @@ -0,0 +1,1803 @@ +/** + * MCP Events: what the server POSTs to a webhook callback — the verification + * handshake, Standard Webhooks signing, retry behaviour, control envelopes and + * the SSRF rules. + * + * Scored against the merged design sketch on `main` of + * modelcontextprotocol/experimental-ext-triggers-events. Each check's verbatim + * excerpt lives beside its id in src/seps/sep-3415.yaml. + * + * **This scenario needs a callback the server under test can reach.** Set + * `EVENTS_WEBHOOK_CALLBACK_BASE` to a public https base URL forwarding to this + * harness (a tunnel, say), and the receiver binds to `0.0.0.0` behind it. + * + * With no such URL the scenario still runs, pointed at a loopback receiver, + * and that is not a degraded mode so much as a different question. The + * document requires a server to refuse a callback whose resolved IP is not + * globally routable, so a loopback URL is exactly the SSRF probe: a server + * that refuses it passes the SSRF rows and reports the delivery rows as + * untestable, while one that happily POSTs to 127.0.0.1 fails the SSRF rows + * and hands the harness real deliveries to grade everything else against. Both + * outcomes are informative, and neither is a false green. + * + * The signature is verified over the raw bytes, per the document's own + * instruction to receivers, so the scenario cannot accidentally pass a server + * that signs a re-serialization of its own JSON. + */ + +import { createHmac, timingSafeEqual } from 'node:crypto'; +import { ClientScenario, ConformanceCheck } from '../../../types'; +import type { Connection, RunContext } from '../../../connection'; +import { JsonRpcError } from '../../../connection'; +import { untestableCheck } from '../../untestable'; +import { + EVENTS_CALLBACK_ENDPOINT_ERROR, + EVENTS_EXTENSION_ID, + extensionsOf, + EVENTS_SPEC_REF, + EVENTS_SUBSCRIBE_METHOD, + EVENTS_UNSUBSCRIBE_METHOD, + JSONRPC_METHOD_NOT_FOUND, + describeValue, + descriptorName, + eventsCheck, + eventsListAll, + EVENTS_CONTROL_ALLOW_CALLBACK_ORIGIN, + EVENTS_CONTROL_WEBHOOK_GAP, + EVENTS_CONTROL_WEBHOOK_TERMINATE, + askControl, + hasControl, + firstSupporting, + isIso8601, + isObject, + minimalArguments +} from './helpers'; +import { + RECEIVER_WELL_KNOWN_PATH, + WRONG_CHALLENGE_ECHO, + startCanary, + startReceiver, + type ReceivedDelivery, + type Receiver +} from './receiver'; + +/** How long to wait for the server to deliver something. */ +const DELIVERY_WAIT_MS = Number(process.env.EVENTS_DELIVERY_WAIT_MS ?? 20000); + +/** + * How long to let retries and the two non-retryable probes play out after the + * first attempt arrives. Retry backoff is the server's to choose, so this is a + * guess at "long enough to see a second attempt"; against a fast local fixture + * it is most of the scenario's wall time, which is why it is a knob. + */ +const SETTLE_MS = Number(process.env.EVENTS_DELIVERY_SETTLE_MS ?? 5000); + +/** + * How long an accepted routability probe is watched for a connection. A server + * that verifies inside events/subscribe has already dialled by the time it + * answers; this covers one that verifies or delivers just after. + */ +const CANARY_WAIT_MS = Math.min(DELIVERY_WAIT_MS, 3000); + +/** How long to wait for a gap or terminated envelope a control asked for. */ +const ENVELOPE_WAIT_MS = Math.min(DELIVERY_WAIT_MS, 5000); + +/** A public base URL forwarding to this harness, when one exists. */ +const PUBLIC_BASE = process.env.EVENTS_WEBHOOK_CALLBACK_BASE; + +/** + * Whether the receiver publishes `/.well-known/mcp-webhook-receiver.json`. + * + * The document is the fourth way the spec lets a server confirm a callback's + * intent, and an origin that serves it needs no challenge POST. Publishing it by + * default costs nothing against a server that only implements the handshake, and + * stops one that implements the document from failing a rule it satisfies. Set + * `EVENTS_RECEIVER_WELL_KNOWN=0` to withhold it and force the handshake path. + */ +const PUBLISH_WELL_KNOWN = process.env.EVENTS_RECEIVER_WELL_KNOWN !== '0'; + +/** + * The one path prefix the well-known document declares. + * + * Deliberately not `/`: the probe paths sit outside it so they still take the + * handshake. The document is also honoured only on an `https` origin, so against + * a loopback receiver this path is never taken and the handshake decides + * everything, the same way the SSRF rows only mean something over a tunnel. + */ +const WELL_KNOWN_PREFIX = '/wk/'; + +/** + * The `lastError` categories, which SEP-3415 closes: `lastError` "MUST be ... + * one of" these, and `-32027`'s `data.reason` is "one of the `lastError` + * categories". The `-32027` row still only warns on an unlisted reason, since + * that sentence carries no keyword. + */ +const LAST_ERROR_CATEGORIES = [ + 'connection_refused', + 'timeout', + 'tls_error', + 'http_4xx', + 'http_5xx', + 'challenge_failed' +]; + +/** 256 KiB, the body-size ceiling the document asks servers to respect. */ +const BODY_CEILING_BYTES = 256 * 1024; + +const DELIVERY_IDS = [ + 'sep-3415-delivery-post-json', + 'sep-3415-delivery-standard-webhooks-headers', + 'sep-3415-delivery-subscription-id-header', + 'sep-3415-delivery-signature-formula', + 'sep-3415-delivery-retry-regenerates-signature', + 'sep-3415-delivery-dual-sign-on-rotation', + 'sep-3415-delivery-body-size', + 'sep-3415-delivery-413-non-retryable', + 'sep-3415-delivery-410-non-retryable', + 'sep-3415-delivery-retries-bounded', + 'sep-3415-delivery-status-last-error-category' +] as const; + +const VERIFICATION_IDS = [ + 'sep-3415-verification-required-before-delivery', + 'sep-3415-verification-challenge-echo', + 'sep-3415-verification-failure-error', + 'sep-3415-verification-cached-per-principal-url', + 'sep-3415-verification-persisted-for-no-expiry', + 'sep-3415-verification-uses-ssrf-hardened-path', + 'sep-3415-verification-no-raw-endpoint-responses', + 'sep-3415-server-identity-key-discovery' +] as const; + +const SSRF_IDS = [ + 'sep-3415-ssrf-validate-callback-url', + 'sep-3415-ssrf-reject-non-routable', + 'sep-3415-ssrf-validate-at-delivery-time', + 'sep-3415-ssrf-no-redirects' +] as const; + +const ENVELOPE_IDS = [ + 'sep-3415-envelope-type-discriminator', + 'sep-3415-envelope-signed-like-deliveries', + 'sep-3415-envelope-webhook-id-format', + 'sep-3415-envelope-gap', + 'sep-3415-envelope-terminated' +] as const; + +/** + * The error-table row this scenario claims. `-32027` is webhook-only and the + * wrong-challenge probe is the only place the suite provokes it, so the row is + * graded there and listed here for the same reason the others are: a path that + * bails early must still report it, rather than emit one row fewer. + */ +const ERROR_IDS = ['sep-3415-error-callback-endpoint-error'] as const; + +const ALL_IDS = [ + ...DELIVERY_IDS, + ...VERIFICATION_IDS, + ...SSRF_IDS, + ...ENVELOPE_IDS, + ...ERROR_IDS +]; + +function untestableAll( + ids: readonly string[], + reason: string, + severity: 'FAILURE' | 'WARNING' = 'FAILURE' +): ConformanceCheck[] { + return ids.map((id) => + untestableCheck(id, id, id, reason, [EVENTS_SPEC_REF], severity) + ); +} + +function skipAll(reason: string): ConformanceCheck[] { + return ALL_IDS.map((id) => + eventsCheck(id, id, 'SKIPPED', { errorMessage: reason }) + ); +} + +function freshSecret(): { value: string; bytes: Buffer } { + const bytes = Buffer.alloc(32); + for (let i = 0; i < bytes.length; i++) { + bytes[i] = Math.floor(Math.random() * 256); + } + return { value: `whsec_${bytes.toString('base64')}`, bytes }; +} + +/** `HMAC-SHA256(secret, id + "." + timestamp + "." + body)`, base64, `v1,`. */ +function expectedSignature( + secret: Buffer, + webhookId: string, + timestamp: string, + rawBody: string +): string { + const mac = createHmac('sha256', secret) + .update(`${webhookId}.${timestamp}.${rawBody}`) + .digest('base64'); + return `v1,${mac}`; +} + +function signatureMatches(header: string, expected: string): boolean { + // The header may carry several space-delimited signatures during rotation. + return header + .split(/\s+/) + .filter(Boolean) + .some((candidate) => { + const a = Buffer.from(candidate); + const b = Buffer.from(expected); + return a.length === b.length && timingSafeEqual(a, b); + }); +} + +export class EventsWebhookDeliveryScenario implements ClientScenario { + name = 'events-webhook-delivery'; + readonly source = { extensionId: EVENTS_EXTENSION_ID } as const; + description = `MCP Events: webhook delivery, signing, verification and the SSRF rules. + +**Methods**: \`events/subscribe\` and \`events/unsubscribe\`, plus an HTTP receiver the harness runs and the server POSTs to + +**Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-3415.yaml): + +- \`sep-3415-verification-*\` — the challenge handshake that must precede delivery, and what a failed one reports +- \`sep-3415-delivery-*\` — POST with JSON, the Standard Webhooks headers plus \`X-MCP-Subscription-Id\`, the signature formula over the raw body, retries and their bounds +- \`sep-3415-envelope-*\` — the \`type\` discriminator, \`msg__\` ids, and the gap and terminated envelopes +- \`sep-3415-ssrf-*\` — callback URLs are validated, non-routable addresses refused, and redirects not followed + +**Needs a reachable callback**: set \`EVENTS_WEBHOOK_CALLBACK_BASE\` to a public https base URL that forwards to this harness. + +**Without one, the loopback receiver is the SSRF probe.** A server that refuses \`http://127.0.0.1\` passes \`validate-callback-url\` and reports the delivery rows untestable; a server that delivers there fails it and supplies real deliveries for everything else. \`reject-non-routable\` is graded separately in either mode, from an \`https://127.0.0.1\` callback aimed at a listener that records whether the server connected, since the scheme rule alone refuses the http probe.`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect(); + let receiver: Receiver | undefined; + try { + const capabilities = await conn.discover(); + const caps = isObject(capabilities.capabilities) + ? capabilities.capabilities + : {}; + const declared = extensionsOf(caps)[EVENTS_EXTENSION_ID] !== undefined; + + const listed = await eventsListAll(conn); + if ('error' in listed) { + if (!declared && listed.error.code === JSONRPC_METHOD_NOT_FOUND) { + return skipAll( + 'Server does not declare the `events` capability and does not implement `events/list`; the extension is optional.' + ); + } + return untestableAll( + ALL_IDS, + `\`events/list\` failed (${listed.error.code} ${listed.error.message}), so no webhook-capable event type could be discovered.` + ); + } + + const target = firstSupporting(listed.descriptors, 'webhook'); + const name = target ? descriptorName(target) : undefined; + if (!target || !name) { + return untestableAll( + ALL_IDS, + 'No event type advertises `webhook` delivery, so nothing could be delivered.' + ); + } + const args = minimalArguments(target); + if (args === undefined) { + return untestableAll( + ALL_IDS, + `Event type \`${name}\` declares required \`inputSchema\` properties the harness cannot satisfy from the schema.` + ); + } + + receiver = await startReceiver(PUBLIC_BASE ? '0.0.0.0' : '127.0.0.1'); + // Only the main delivery path is declared, never the probe paths. A + // server that reads the document verifies that one without a challenge, + // and still has to handshake for everything under a prefix the document + // does not name, so one run exercises both consent paths instead of + // whichever the server happens to prefer. + if (PUBLISH_WELL_KNOWN) receiver.publishWellKnown([WELL_KNOWN_PREFIX]); + return await this.deliveryChecks(conn, receiver, name, args); + } finally { + await receiver?.close(); + await conn.close(); + } + } + + private callbackFor(receiver: Receiver, path: string): string { + return PUBLIC_BASE + ? `${PUBLIC_BASE.replace(/\/$/, '')}${path}` + : `${receiver.url}${path}`; + } + + private async deliveryChecks( + conn: Connection, + receiver: Receiver, + name: string, + args: Record + ): Promise { + const checks: ConformanceCheck[] = []; + const path = `${WELL_KNOWN_PREFIX}hook-${Date.now()}`; + const url = this.callbackFor(receiver, path); + const secret = freshSecret(); + + const subscribe = async ( + callbackUrl: string + ): Promise<{ id?: unknown } | { error: JsonRpcError }> => { + try { + const result = await conn.request<{ id?: unknown }>( + EVENTS_SUBSCRIBE_METHOD, + { + name, + arguments: args, + delivery: { + mode: 'webhook', + url: callbackUrl, + secret: secret.value + }, + cursor: null, + ttlMs: 3600_000 + } + ); + return result ?? {}; + } catch (err) { + if (err instanceof JsonRpcError) return { error: err }; + throw err; + } + }; + const release = async (callbackUrl: string): Promise => { + try { + await conn.request(EVENTS_UNSUBSCRIBE_METHOD, { + name, + arguments: args, + delivery: { mode: 'webhook', url: callbackUrl } + }); + } catch { + // Already gone. + } + }; + + // Routability first, on its own probe, before anything below can lift a + // guard. dedupe keeps the first row per id, so this one is authoritative. + checks.push(await this.nonRoutableCheck(subscribe, release)); + + let subscribed = await subscribe(url); + + // --- The SSRF rows, which a loopback callback answers directly --------- + const loopback = !PUBLIC_BASE; + let guardLifted = false; + if ('error' in subscribed && loopback) { + // Graded here, before anything is lifted, so the verdict is about the + // server as configured rather than as persuaded. + checks.push(...this.ssrfRefusedChecks(subscribed.error)); + guardLifted = await this.liftCallbackGuard(conn, receiver); + if (guardLifted) subscribed = await subscribe(url); + } + if ('error' in subscribed) { + const refused = subscribed.error; + if (loopback) { + checks.push( + ...untestableAll( + [ + ...DELIVERY_IDS, + ...VERIFICATION_IDS, + ...ENVELOPE_IDS, + ...ERROR_IDS, + // The two SSRF rows the refusal does not answer by itself: one + // needs a delivery to revalidate, the other a redirect to refuse. + 'sep-3415-ssrf-validate-at-delivery-time', + 'sep-3415-ssrf-no-redirects' + ], + guardLifted + ? `The server still refused ${url} after \`${EVENTS_CONTROL_ALLOW_CALLBACK_ORIGIN}\` was called for its origin (${refused.code} ${refused.message}), so nothing could be delivered.` + : `The server refused a loopback callback (${refused.code} ${refused.message}), which is what the SSRF rules ask of it. Grading delivery needs a callback it will accept: set EVENTS_WEBHOOK_CALLBACK_BASE to a public https URL forwarding to this harness, or expose the \`${EVENTS_CONTROL_ALLOW_CALLBACK_ORIGIN}\` control so this one origin is permitted after the SSRF rows are graded.` + ) + ); + return dedupe(checks); + } + checks.push( + eventsCheck( + 'sep-3415-delivery-post-json', + 'Deliveries are HTTP `POST` only, with Content-Type `application/json`.', + 'FAILURE', + { + errorMessage: `Subscribing with the configured callback ${url} answered ${refused.code} ${refused.message}, so nothing could be delivered.` + } + ) + ); + checks.push( + ...untestableAll( + ALL_IDS.filter((id) => id !== 'sep-3415-delivery-post-json'), + `No subscription could be created against ${url}.` + ) + ); + return dedupe(checks); + } + + try { + const subscriptionId = subscribed.id; + + // Wait for whatever the server sends: the verification challenge first, + // then events. + const first = await receiver.waitFor(path, () => true, DELIVERY_WAIT_MS); + // One delivery is a thin sample for the header and signature rows, so + // give a server emitting on a cadence a moment to send a few more. + if (first) + await new Promise((resolve) => + setTimeout(resolve, Math.min(3000, SETTLE_MS)) + ); + const all = receiver.on(path); + + if (loopback && !guardLifted) { + checks.push(...this.ssrfDeliveredChecks(all.length > 0, url)); + } else if (loopback) { + // Already graded above, against the default configuration. Deliveries + // here happened because this origin was permitted on purpose, so they + // say nothing about the rule. + checks.push( + eventsCheck( + 'sep-3415-ssrf-validate-at-delivery-time', + 'To prevent DNS rebinding, validation MUST be performed at delivery time, not only at subscribe time.', + 'SKIPPED', + { + errorMessage: `This origin was permitted through \`${EVENTS_CONTROL_ALLOW_CALLBACK_ORIGIN}\`, so a delivery to it is not evidence either way. Proving revalidation needs a hostname whose DNS answer changes between subscribe and delivery.` + } + ) + ); + } else { + checks.push( + ...untestableAll( + ['sep-3415-ssrf-validate-callback-url'], + 'The configured callback is routable, so the refusal path was not exercised. Run without EVENTS_WEBHOOK_CALLBACK_BASE to probe it with a loopback URL.', + 'WARNING' + ) + ); + } + checks.push( + untestableCheck( + 'sep-3415-ssrf-validate-at-delivery-time', + 'sep-3415-ssrf-validate-at-delivery-time', + 'To prevent DNS rebinding, validation MUST be performed at delivery time, not only at subscribe time.', + 'Proving delivery-time revalidation needs a hostname whose DNS answer changes between subscribe and delivery, which the harness cannot serve.', + [EVENTS_SPEC_REF] + ) + ); + + if (!first) { + checks.push( + ...untestableAll( + [ + ...DELIVERY_IDS, + ...VERIFICATION_IDS, + ...ENVELOPE_IDS, + ...ERROR_IDS + ].filter( + (id) => id !== 'sep-3415-delivery-status-last-error-category' + ), + `Nothing arrived at ${url} within ${DELIVERY_WAIT_MS}ms of subscribing, so no delivery could be graded.` + ) + ); + checks.push( + untestableCheck( + 'sep-3415-delivery-status-last-error-category', + 'sep-3415-delivery-status-last-error-category', + '`lastError` MUST be a server-generated category string and MUST NOT include raw response bodies.', + 'No delivery was attempted, so no `deliveryStatus.lastError` could be observed. The field is OPTIONAL in any case.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + checks.push( + untestableCheck( + 'sep-3415-ssrf-no-redirects', + 'sep-3415-ssrf-no-redirects', + 'Webhook delivery requests MUST NOT follow HTTP redirects.', + 'No delivery was attempted, so the redirect probe had nothing to redirect.', + [EVENTS_SPEC_REF] + ) + ); + return dedupe(checks); + } + + checks.push( + ...this.verificationChecks( + all, + subscriptionId, + receiver.wellKnownFetches() + ) + ); + // The residual rows come last because one of them grades what the server + // said when the handshake failed, which only the probe below produces. + let endpointFailure: JsonRpcError | undefined; + if (all.some(isVerificationEnvelope)) { + const failed = await this.verificationFailureCheck( + receiver, + subscribe, + release + ); + checks.push(failed.check); + endpointFailure = failed.serverError; + } + checks.push( + ...this.verificationResidualChecks(subscriptionId, endpointFailure) + ); + checks.push(this.callbackEndpointErrorCheck(endpointFailure)); + checks.push(...this.transportChecks(all, subscriptionId)); + checks.push(...this.signatureChecks(all, secret.bytes)); + checks.push(...(await this.redirectChecks(receiver, subscribe, release))); + // Whether the source fires more than once, so the 410 probe can tell a + // suspended subscription from a quiet event. + const periodic = + new Set( + receiver + .on(path) + .filter(isEvent) + .map((d) => d.headers['webhook-id']) + ).size >= 2; + checks.push( + ...(await this.retryChecks(receiver, subscribe, release, periodic)) + ); + // Last, because terminating ends the subscription everything above + // delivered to. The envelope rows grade from what arrived after, so the + // signalled gap and terminated envelopes count toward the discriminator, + // signing and id rows too. + const signalled = await this.signalEnvelopes( + conn, + receiver, + path, + subscriptionId + ); + checks.push(...this.envelopeChecks(receiver.on(path), signalled)); + return dedupe(checks); + } finally { + await release(url); + } + } + + /** + * Ask the server to permit this receiver's origin, after the SSRF rows have + * been graded against its default configuration. + * + * Absent control means no override, which is the normal case and never an + * error: the delivery rows then report untestable and name both ways out. + * Returns whether the server acknowledged, not whether the next subscribe will + * succeed, because a server may decline for its own reasons. + */ + private async liftCallbackGuard( + conn: Connection, + receiver: Receiver + ): Promise { + if (!(await hasControl(conn, EVENTS_CONTROL_ALLOW_CALLBACK_ORIGIN))) { + return false; + } + const origin = new URL(receiver.url).origin; + const answer = await askControl( + conn, + EVENTS_CONTROL_ALLOW_CALLBACK_ORIGIN, + { origin } + ); + return answer !== undefined; + } + + /** + * The server refused the loopback http callback. Either the scheme rule or + * routability can do that, so it answers "validates callback URLs" and + * nothing narrower; nonRoutableCheck grades routability on a URL the scheme + * rule cannot refuse. + */ + private ssrfRefusedChecks(error: JsonRpcError): ConformanceCheck[] { + return [ + eventsCheck( + 'sep-3415-ssrf-validate-callback-url', + 'The server MUST validate callback URLs.', + 'SUCCESS', + { details: { code: error.code, message: error.message } } + ) + ]; + } + + /** + * `reject-non-routable`, graded from an `https://127.0.0.1` callback aimed at + * a canary listener. The https scheme takes the scheme rule out of the + * question, and the canary says whether the server dialled, which is what + * separates "refused because the address is not routable" from "tried it and + * the handshake failed". + */ + private async nonRoutableCheck( + subscribe: ( + callbackUrl: string + ) => Promise<{ id?: unknown } | { error: JsonRpcError }>, + release: (callbackUrl: string) => Promise + ): Promise { + const id = 'sep-3415-ssrf-reject-non-routable'; + const description = + 'Servers SHOULD reject URLs whose resolved IP is not globally routable per the IANA Special-Purpose Address Registries.'; + const canary = await startCanary(); + const probe = `https://127.0.0.1:${canary.port}/ssrf-probe-${Date.now()}`; + try { + const answered = await subscribe(probe); + const refused = 'error' in answered ? answered.error : undefined; + if (!refused) { + const deadline = Date.now() + CANARY_WAIT_MS; + while (canary.connections() === 0 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + await release(probe); + } + const dialled = canary.connections(); + + if (dialled > 0) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: refused + ? `The server answered ${refused.code} ${refused.message} for ${probe}, but only after connecting to it (${dialled} connection(s)). A callback on 127.0.0.0/8 was dialled rather than refused, so the error came from the connection failing, not from the address check.` + : `The server accepted ${probe} and connected to it (${dialled} connection(s)). 127.0.0.0/8 is not globally routable, so a caller can aim requests at services the server can reach and the caller cannot.`, + details: { url: probe, connections: dialled, code: refused?.code } + }); + } + if (!refused) { + return eventsCheck(id, description, 'WARNING', { + errorMessage: `${probe} was accepted at subscribe time and nothing connected to it within ${CANARY_WAIT_MS}ms. That may be delivery-time validation rather than a missing check, and an idle event type looks the same.`, + details: { url: probe } + }); + } + // Over a tunnel the server may not share this host, so its dial to + // 127.0.0.1 would never reach the canary. Only a parameter error is + // unambiguous then. + if (PUBLIC_BASE && refused.code !== -32602) { + return eventsCheck(id, description, 'WARNING', { + errorMessage: `${probe} was refused with ${refused.code} ${refused.message}. With EVENTS_WEBHOOK_CALLBACK_BASE set the server may be on another host, where a failed connection and an address refusal look the same from here; only -32602 settles it.`, + details: { url: probe, code: refused.code, message: refused.message } + }); + } + return eventsCheck(id, description, 'SUCCESS', { + details: { + url: probe, + code: refused.code, + message: refused.message, + connections: 0 + } + }); + } finally { + await canary.close(); + } + } + + /** + * The server accepted a loopback callback. Did it also deliver there? Only + * `validate-callback-url` is graded here; routability has its own probe. + */ + private ssrfDeliveredChecks( + delivered: boolean, + url: string + ): ConformanceCheck[] { + if (!delivered) { + return [ + eventsCheck( + 'sep-3415-ssrf-validate-callback-url', + 'The server MUST validate callback URLs.', + 'WARNING', + { + errorMessage: `The subscribe for ${url} was accepted, but nothing was delivered, so the harness cannot tell delivery-time hardening from an idle event type.` + } + ) + ]; + } + return [ + eventsCheck( + 'sep-3415-ssrf-validate-callback-url', + 'The server MUST validate callback URLs.', + 'FAILURE', + { + errorMessage: `The server POSTed to ${url}, a loopback address. A callback URL pointing inside the server's own network was neither refused at subscribe time nor at delivery time.` + } + ) + ]; + } + + /** The challenge handshake that must precede any event delivery. */ + private verificationChecks( + all: ReceivedDelivery[], + subscriptionId: unknown, + wellKnownFetches: number + ): ConformanceCheck[] { + const out: ConformanceCheck[] = []; + const verification = all.find(isVerificationEnvelope); + const events = all.filter((d) => typeof d.json?.eventId === 'string'); + + // The document names four ways to confirm intent, and two of them are + // invisible from here: a server-configured allowlist and prior out-of-band + // verification. Neither can apply to these callbacks, because the path is + // minted fresh for the run and no operator has ever seen it, so a server + // that delivers to it has either handshaken, read the well-known document, + // or skipped consent. That is what keeps the failure below honest. + if (!verification && wellKnownFetches > 0) { + out.push( + eventsCheck( + 'sep-3415-verification-required-before-delivery', + "A server MUST NOT begin delivering to a callback URL until the endpoint's intent to receive deliveries is confirmed, by one of: a verification handshake, a server-configured allowlist, prior out-of-band verification, or a receiver-published well-known document.", + 'SUCCESS', + { + details: { + via: RECEIVER_WELL_KNOWN_PATH, + fetches: wellKnownFetches, + note: 'The origin published its consent, so no challenge POST is required.' + } + } + ) + ); + out.push( + ...untestableAll( + [ + 'sep-3415-verification-challenge-echo', + 'sep-3415-verification-failure-error' + ], + `The server confirmed intent by fetching ${RECEIVER_WELL_KNOWN_PATH} and sent no challenge, so the echo path was not exercised. Re-run with EVENTS_RECEIVER_WELL_KNOWN=0 to withhold the document and force the handshake.`, + 'WARNING' + ) + ); + return out; + } + + if (!verification) { + out.push( + eventsCheck( + 'sep-3415-verification-required-before-delivery', + "A server MUST NOT begin delivering to a callback URL until the endpoint's intent to receive deliveries is confirmed.", + events.length > 0 ? 'FAILURE' : 'WARNING', + { + errorMessage: + events.length > 0 + ? `${events.length} event(s) were delivered to a callback that was never asked to prove intent. An attacker can aim a subscription at a third party and the server will POST to it.` + : 'No verification envelope and no events arrived, so intent confirmation could not be observed.', + details: { deliveries: all.length } + } + ) + ); + out.push( + ...untestableAll( + [ + 'sep-3415-verification-challenge-echo', + 'sep-3415-verification-failure-error' + ], + 'The server sent no verification challenge, so the echo path could not be exercised.' + ) + ); + } else { + out.push( + eventsCheck( + 'sep-3415-verification-required-before-delivery', + "A server MUST NOT begin delivering to a callback URL until the endpoint's intent to receive deliveries is confirmed.", + events.length === 0 || + verification.atMs <= (events[0]?.atMs ?? Infinity) + ? 'SUCCESS' + : 'FAILURE', + { + errorMessage: + events.length > 0 && + verification.atMs > (events[0]?.atMs ?? Infinity) + ? 'An event was delivered before the verification challenge.' + : undefined, + details: { verificationAtMs: verification.atMs } + } + ) + ); + out.push( + eventsCheck( + 'sep-3415-verification-challenge-echo', + 'Before activating, the server POSTs a `verification` control envelope carrying a single-use, short-lived `challenge` nonce, and the endpoint proves intent by echoing it in a 2xx body.', + typeof verification.json?.challenge === 'string' + ? 'SUCCESS' + : 'FAILURE', + { + errorMessage: + typeof verification.json?.challenge === 'string' + ? undefined + : `The verification envelope carried \`challenge\` ${describeValue(verification.json?.challenge)}, expected a string nonce.`, + details: { body: verification.json } + } + ) + ); + } + + return out; + } + + /** + * The verification rows that do not depend on which consent path the server + * took, kept in one place so every branch above emits the same row set. A + * branch that emitted fewer would read as a shorter suite rather than as a + * prerequisite that was missing. + */ + private verificationResidualChecks( + subscriptionId: unknown, + endpointFailure?: JsonRpcError + ): ConformanceCheck[] { + const out: ConformanceCheck[] = []; + out.push( + untestableCheck( + 'sep-3415-verification-uses-ssrf-hardened-path', + 'sep-3415-verification-uses-ssrf-hardened-path', + 'The verification POST MUST use the same SSRF-hardened path as deliveries.', + 'Both the verification POST and the deliveries are observed from the receiver, so they cannot be distinguished as using the same internal code path. The SSRF rows grade the outcome the rule protects.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + out.push( + ...untestableAll( + [ + 'sep-3415-verification-cached-per-principal-url', + 'sep-3415-verification-persisted-for-no-expiry', + 'sep-3415-server-identity-key-discovery' + ], + 'Needs a second principal, a server restart, or a key-discovery origin the harness does not control.', + 'WARNING' + ) + ); + out.push(this.noRawEndpointResponsesCheck(subscriptionId, endpointFailure)); + return out; + } + + /** + * The `-32027` row from the error table, graded off the same probe as + * sep-3415-verification-failure-error rather than a second failing callback. + * That row grades the rule; this one grades the code and the `data.reason` + * category the table requires of it. + */ + private callbackEndpointErrorCheck( + endpointFailure?: JsonRpcError + ): ConformanceCheck { + const id = 'sep-3415-error-callback-endpoint-error'; + const description = + '`-32027 CallbackEndpointError` — a client-supplied callback endpoint failed verification or could not be reached (webhook mode only). `data.reason` is one of the `lastError` categories.'; + if (!endpointFailure) { + return untestableCheck( + id, + id, + description, + 'No callback failed verification or connection during this run, so the code was never provoked. A server that implements the verification handshake answers it for the wrong-challenge probe.', + [EVENTS_SPEC_REF] + ); + } + const reason = isObject(endpointFailure.data) + ? endpointFailure.data.reason + : undefined; + if (endpointFailure.code !== EVENTS_CALLBACK_ENDPOINT_ERROR) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `A callback that failed verification answered ${endpointFailure.code} ${endpointFailure.message}, where the table names ${EVENTS_CALLBACK_ENDPOINT_ERROR} CallbackEndpointError.`, + details: { code: endpointFailure.code, data: endpointFailure.data } + }); + } + return LAST_ERROR_CATEGORIES.includes(String(reason)) + ? eventsCheck(id, description, 'SUCCESS', { + details: { + code: endpointFailure.code, + reason, + gradedBy: 'sep-3415-verification-failure-error' + } + }) + : eventsCheck(id, description, 'WARNING', { + errorMessage: `\`${EVENTS_CALLBACK_ENDPOINT_ERROR}\` carried \`data.reason\` ${describeValue(reason)}, which is not one of the documented \`lastError\` categories (${LAST_ERROR_CATEGORIES.join(', ')}).`, + details: { reason } + }); + } + + /** + * Whether a failed handshake leaked the endpoint's own response. + * + * Only gradeable when something actually failed, which is why it waits for the + * wrong-challenge probe rather than passing on the strength of a quiet run. + * The receiver echoes a distinctive string, so finding it in the server's + * error is proof the body was passed through to the subscriber. + */ + private noRawEndpointResponsesCheck( + subscriptionId: unknown, + endpointFailure?: JsonRpcError + ): ConformanceCheck { + const id = 'sep-3415-verification-no-raw-endpoint-responses'; + const description = + 'Failures surface only via the `lastError` category `challenge_failed`, never raw endpoint responses.'; + if (!endpointFailure) { + return untestableCheck( + id, + id, + description, + 'No endpoint failed verification during this run, so nothing could have carried its response body back. The wrong-challenge probe supplies one against a server that implements the handshake.', + [EVENTS_SPEC_REF], + 'WARNING' + ); + } + const reported = JSON.stringify({ + message: endpointFailure.message, + data: endpointFailure.data + }); + return reported.includes(WRONG_CHALLENGE_ECHO) + ? eventsCheck(id, description, 'FAILURE', { + errorMessage: `The error for a failed handshake carried the endpoint's own response (${JSON.stringify(WRONG_CHALLENGE_ECHO)}). A subscriber learns what an arbitrary third-party URL answered, which is the reflection the category exists to avoid.`, + details: { reported } + }) + : eventsCheck(id, description, 'SUCCESS', { + details: { subscriptionId, code: endpointFailure.code } + }); + } + + /** POST, content type, and the headers every delivery must carry. */ + private transportChecks( + all: ReceivedDelivery[], + subscriptionId: unknown + ): ConformanceCheck[] { + const out: ConformanceCheck[] = []; + const graded = all.filter((d) => d.respondedStatus !== 302); + + const badMethod = graded.filter((d) => d.method !== 'POST'); + const badType = graded.filter( + (d) => !(d.headers['content-type'] ?? '').includes('application/json') + ); + out.push( + badMethod.length === 0 && badType.length === 0 + ? eventsCheck( + 'sep-3415-delivery-post-json', + 'Deliveries are HTTP `POST` only, with Content-Type `application/json`.', + 'SUCCESS', + { details: { deliveries: graded.length } } + ) + : eventsCheck( + 'sep-3415-delivery-post-json', + 'Deliveries are HTTP `POST` only, with Content-Type `application/json`.', + 'FAILURE', + { + errorMessage: [ + badMethod.length + ? `${badMethod.length} delivery(ies) used ${[...new Set(badMethod.map((d) => d.method))].join(', ')}` + : undefined, + badType.length + ? `${badType.length} carried content-type ${[...new Set(badType.map((d) => d.headers['content-type'] ?? 'absent'))].join(', ')}` + : undefined + ] + .filter(Boolean) + .join('; ') + } + ) + ); + + const missingHeaders = graded + .map((d) => ({ + d, + missing: [ + 'webhook-id', + 'webhook-timestamp', + 'webhook-signature' + ].filter((h) => !d.headers[h]) + })) + .filter((x) => x.missing.length > 0); + out.push( + missingHeaders.length === 0 + ? eventsCheck( + 'sep-3415-delivery-standard-webhooks-headers', + 'Every delivery MUST include `webhook-id`, `webhook-timestamp` (Unix seconds), and `webhook-signature`.', + 'SUCCESS', + { details: { deliveries: graded.length } } + ) + : eventsCheck( + 'sep-3415-delivery-standard-webhooks-headers', + 'Every delivery MUST include `webhook-id`, `webhook-timestamp` (Unix seconds), and `webhook-signature`.', + 'FAILURE', + { + errorMessage: `${missingHeaders.length} of ${graded.length} delivery(ies) were missing ${[...new Set(missingHeaders.flatMap((x) => x.missing))].join(', ')}.` + } + ) + ); + + const missingSubId = graded.filter( + (d) => !d.headers['x-mcp-subscription-id'] + ); + const wrongSubId = + typeof subscriptionId === 'string' + ? graded.filter( + (d) => + d.headers['x-mcp-subscription-id'] && + d.headers['x-mcp-subscription-id'] !== subscriptionId + ) + : []; + out.push( + missingSubId.length === 0 && wrongSubId.length === 0 + ? eventsCheck( + 'sep-3415-delivery-subscription-id-header', + 'Deliveries MUST include `X-MCP-Subscription-Id` so the receiver can select the correct secret without parsing the body.', + 'SUCCESS', + { details: { subscriptionId } } + ) + : eventsCheck( + 'sep-3415-delivery-subscription-id-header', + 'Deliveries MUST include `X-MCP-Subscription-Id` so the receiver can select the correct secret without parsing the body.', + 'FAILURE', + { + errorMessage: + missingSubId.length > 0 + ? `${missingSubId.length} of ${graded.length} delivery(ies) carried no \`X-MCP-Subscription-Id\`; a receiver holding several subscriptions must parse the body to pick a secret.` + : `${wrongSubId.length} delivery(ies) carried an \`X-MCP-Subscription-Id\` other than the subscribe response's \`id\` (${String(subscriptionId)}).` + } + ) + ); + + const oversized = graded.filter( + (d) => Buffer.byteLength(d.rawBody, 'utf8') > BODY_CEILING_BYTES + ); + out.push( + oversized.length === 0 + ? eventsCheck( + 'sep-3415-delivery-body-size', + 'Servers SHOULD keep delivery bodies at or under 256 KiB, consistent with Payload Minimality.', + 'SUCCESS', + { + details: { + largestBytes: Math.max( + 0, + ...graded.map((d) => Buffer.byteLength(d.rawBody, 'utf8')) + ) + } + } + ) + : eventsCheck( + 'sep-3415-delivery-body-size', + 'Servers SHOULD keep delivery bodies at or under 256 KiB, consistent with Payload Minimality.', + 'WARNING', + { + errorMessage: `${oversized.length} delivery(ies) exceeded 256 KiB (largest ${Math.max(...oversized.map((d) => Buffer.byteLength(d.rawBody, 'utf8')))} bytes).` + } + ) + ); + + out.push( + untestableCheck( + 'sep-3415-delivery-dual-sign-on-rotation', + 'sep-3415-delivery-dual-sign-on-rotation', + 'The server SHOULD dual-sign deliveries with both the old and new secrets for a short grace window.', + 'Observing it needs a secret rotated while a delivery is in flight, which the harness cannot time reliably.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + return out; + } + + /** The signature, computed over the raw bytes exactly as they arrived. */ + private signatureChecks( + all: ReceivedDelivery[], + secret: Buffer + ): ConformanceCheck[] { + const signed = all.filter( + (d) => d.headers['webhook-signature'] && d.headers['webhook-id'] + ); + if (signed.length === 0) { + return untestableAll( + ['sep-3415-delivery-signature-formula'], + 'No delivery carried both `webhook-id` and `webhook-signature`, so the formula could not be checked.' + ); + } + const bad = signed.filter( + (d) => + !signatureMatches( + d.headers['webhook-signature'], + expectedSignature( + secret, + d.headers['webhook-id'], + d.headers['webhook-timestamp'] ?? '', + d.rawBody + ) + ) + ); + const description = + 'The signature is `HMAC-SHA256(secret, webhook-id + "." + webhook-timestamp + "." + body)` encoded as base64 with a `v1,` prefix, over the raw body bytes.'; + return [ + bad.length === 0 + ? eventsCheck( + 'sep-3415-delivery-signature-formula', + description, + 'SUCCESS', + { details: { verified: signed.length } } + ) + : eventsCheck( + 'sep-3415-delivery-signature-formula', + description, + 'FAILURE', + { + errorMessage: `${bad.length} of ${signed.length} delivery(ies) carried a signature that did not verify over the raw body with the subscription's secret.`, + details: { + firstHeader: bad[0]?.headers['webhook-signature'], + firstId: bad[0]?.headers['webhook-id'], + firstTimestamp: bad[0]?.headers['webhook-timestamp'] + } + } + ) + ]; + } + + /** Control envelopes versus event bodies. */ + /** + * Ask the server to send this subscription a gap, then to end it, and wait + * for each envelope. Absent controls are the normal case; the rows then fall + * back to whatever the server sent unasked. + */ + private async signalEnvelopes( + conn: Connection, + receiver: Receiver, + path: string, + subscriptionId: unknown + ): Promise { + const signal = async ( + tool: string, + type: string + ): Promise => { + if (!(await hasControl(conn, tool))) return 'absent'; + if (typeof subscriptionId !== 'string') return 'refused'; + if ((await askControl(conn, tool, { id: subscriptionId })) === undefined) + return 'refused'; + await receiver.waitFor( + path, + (d) => d.json?.type === type, + ENVELOPE_WAIT_MS + ); + return 'sent'; + }; + const gap = await signal(EVENTS_CONTROL_WEBHOOK_GAP, 'gap'); + const terminate = await signal( + EVENTS_CONTROL_WEBHOOK_TERMINATE, + 'terminated' + ); + return { gap, terminate }; + } + + private envelopeChecks( + all: ReceivedDelivery[], + signalled: SignalledEnvelopes + ): ConformanceCheck[] { + const out: ConformanceCheck[] = []; + const envelopes = all.filter((d) => typeof d.json?.type === 'string'); + const events = all.filter( + (d) => + d.json && + d.json.type === undefined && + typeof d.json.eventId === 'string' + ); + + out.push( + envelopes.length > 0 || events.length > 0 + ? eventsCheck( + 'sep-3415-envelope-type-discriminator', + 'A body with a top-level `type` field is a control envelope; a body without one is an `EventOccurrence`.', + events.every((d) => isIso8601(d.json?.timestamp)) + ? 'SUCCESS' + : 'FAILURE', + { + errorMessage: events.every((d) => isIso8601(d.json?.timestamp)) + ? undefined + : 'A body with no `type` was not a well-formed `EventOccurrence` (its `timestamp` is not ISO 8601).', + details: { envelopes: envelopes.length, events: events.length } + } + ) + : untestableCheck( + 'sep-3415-envelope-type-discriminator', + 'sep-3415-envelope-type-discriminator', + 'A body with a top-level `type` field is a control envelope; a body without one is an `EventOccurrence`.', + 'Nothing with a JSON body arrived, so neither shape was seen.', + [EVENTS_SPEC_REF] + ) + ); + + if (envelopes.length === 0) { + out.push( + ...untestableAll( + [ + 'sep-3415-envelope-signed-like-deliveries', + 'sep-3415-envelope-webhook-id-format' + ], + 'No control envelope arrived during the run.' + ) + ); + } else { + out.push( + envelopes.every( + (d) => + d.headers['webhook-signature'] && d.headers['x-mcp-subscription-id'] + ) + ? eventsCheck( + 'sep-3415-envelope-signed-like-deliveries', + 'Control envelopes are signed and headed exactly like event deliveries.', + 'SUCCESS', + { details: { envelopes: envelopes.length } } + ) + : eventsCheck( + 'sep-3415-envelope-signed-like-deliveries', + 'Control envelopes are signed and headed exactly like event deliveries.', + 'FAILURE', + { + errorMessage: + 'A control envelope arrived without the full Standard Webhooks header set plus `X-MCP-Subscription-Id`, so a receiver cannot verify it the way it verifies events.' + } + ) + ); + const badIds = envelopes.filter((d) => { + const id = d.headers['webhook-id'] ?? ''; + return !/^msg_[a-z]+_.+/i.test(id); + }); + out.push( + badIds.length === 0 + ? eventsCheck( + 'sep-3415-envelope-webhook-id-format', + '`webhook-id` for control envelopes is a per-message identifier of the form `msg__` so receivers can dedup retries.', + 'SUCCESS', + { + details: { ids: envelopes.map((d) => d.headers['webhook-id']) } + } + ) + : eventsCheck( + 'sep-3415-envelope-webhook-id-format', + '`webhook-id` for control envelopes is a per-message identifier of the form `msg__`.', + 'WARNING', + { + errorMessage: `${badIds.length} control envelope(s) carried a \`webhook-id\` outside the documented form (e.g. ${describeValue(badIds[0]?.headers['webhook-id'])}).` + } + ) + ); + } + + // A gap and a termination cannot be provoked from the client side. The + // webhook controls provoke one each on this subscription; without them the + // rows are still graded when a server sends one unasked. + const gapDesc = + 'A `gap` envelope `{"type":"gap","cursor":""}` is sent when a gap is detected between refreshes. The client persists `cursor` and treats it as `truncated: true`.'; + const gap = all.find((d) => d.json?.type === 'gap'); + out.push( + !gap + ? signalled.gap === 'sent' + ? eventsCheck('sep-3415-envelope-gap', gapDesc, 'WARNING', { + errorMessage: `\`${EVENTS_CONTROL_WEBHOOK_GAP}\` acknowledged, and no \`gap\` envelope arrived within ${ENVELOPE_WAIT_MS}ms.` + }) + : untestableCheck( + 'sep-3415-envelope-gap', + 'sep-3415-envelope-gap', + gapDesc, + signalled.gap === 'refused' + ? `No retention gap occurred during the run, and \`${EVENTS_CONTROL_WEBHOOK_GAP}\` declined to signal one.` + : `No retention gap occurred during the run, and the harness cannot force one from the client side. Needs a fixture exposing the \`${EVENTS_CONTROL_WEBHOOK_GAP}\` control.`, + [EVENTS_SPEC_REF], + 'WARNING' + ) + : eventsCheck( + 'sep-3415-envelope-gap', + gapDesc, + typeof gap.json?.cursor === 'string' ? 'SUCCESS' : 'WARNING', + { + errorMessage: + typeof gap.json?.cursor === 'string' + ? undefined + : `A \`gap\` envelope carried \`cursor\` ${describeValue(gap.json?.cursor)}; without a fresh position the client has nothing to persist.`, + details: { body: gap.json } + } + ) + ); + + const terminatedDesc = + 'A `terminated` envelope `{"type":"terminated","error":{...}}` is sent when the subscription has ended (e.g., authorization revoked). The subscription no longer exists server-side.'; + const terminated = all.find((d) => d.json?.type === 'terminated'); + out.push( + !terminated + ? signalled.terminate === 'sent' + ? eventsCheck( + 'sep-3415-envelope-terminated', + terminatedDesc, + 'FAILURE', + { + errorMessage: `\`${EVENTS_CONTROL_WEBHOOK_TERMINATE}\` acknowledged ending the subscription, and no \`terminated\` envelope arrived within ${ENVELOPE_WAIT_MS}ms, so the receiver was never told.` + } + ) + : untestableCheck( + 'sep-3415-envelope-terminated', + 'sep-3415-envelope-terminated', + terminatedDesc, + signalled.terminate === 'refused' + ? `The subscription was not terminated during the run, and \`${EVENTS_CONTROL_WEBHOOK_TERMINATE}\` declined to end it.` + : `The subscription was not terminated during the run. Needs a server that can revoke authorization or remove an event type mid-run, or a fixture exposing the \`${EVENTS_CONTROL_WEBHOOK_TERMINATE}\` control.`, + [EVENTS_SPEC_REF], + 'WARNING' + ) + : eventsCheck( + 'sep-3415-envelope-terminated', + terminatedDesc, + isObject(terminated.json?.error) ? 'SUCCESS' : 'WARNING', + { + errorMessage: isObject(terminated.json?.error) + ? undefined + : `A \`terminated\` envelope carried \`error\` ${describeValue(terminated.json?.error)}; without it the client cannot tell revocation from removal.`, + details: { body: terminated.json } + } + ) + ); + return out; + } + + /** + * A callback that answers the challenge with the wrong nonce. The document + * has the failure come back from events/subscribe itself as -32027 with + * `data.reason: "challenge_failed"`. A server that accepts instead and then + * delivers there has sent events to an endpoint that never consented; one + * that accepts and withholds delivery verified asynchronously, which keeps + * the endpoint safe but reports nothing to the subscriber. + */ + private async verificationFailureCheck( + receiver: Receiver, + subscribe: ( + url: string + ) => Promise<{ id?: unknown } | { error: JsonRpcError }>, + release: (url: string) => Promise + ): Promise<{ check: ConformanceCheck; serverError?: JsonRpcError }> { + const id = 'sep-3415-verification-failure-error'; + const description = + 'A reachable endpoint that fails to echo yields `-32027 CallbackEndpointError` with `data.reason: "challenge_failed"`.'; + const path = `/wrong-challenge-${Date.now()}`; + receiver.behave(path, { kind: 'wrong-challenge' }); + const url = this.callbackFor(receiver, path); + const probe = await subscribe(url); + + if ('error' in probe) { + const { code, message, data } = probe.error; + const reason = isObject(data) ? data.reason : undefined; + if ( + code === EVENTS_CALLBACK_ENDPOINT_ERROR && + reason === 'challenge_failed' + ) { + return { + check: eventsCheck(id, description, 'SUCCESS', { + details: { code, reason } + }), + serverError: probe.error + }; + } + return { + check: eventsCheck(id, description, 'FAILURE', { + errorMessage: `Subscribing a callback that echoed the wrong nonce answered ${code} ${message} with data.reason ${describeValue(reason)}, expected ${EVENTS_CALLBACK_ENDPOINT_ERROR} with "challenge_failed".`, + details: { code, data } + }), + serverError: probe.error + }; + } + + try { + const delivered = await receiver.waitFor( + path, + (d) => typeof d.json?.eventId === 'string', + DELIVERY_WAIT_MS + ); + if (delivered) { + return { + check: eventsCheck(id, description, 'FAILURE', { + errorMessage: `The subscribe succeeded although the callback echoed the wrong nonce, and an event was then delivered to it. The document has this refused with ${EVENTS_CALLBACK_ENDPOINT_ERROR} "challenge_failed" from events/subscribe, and no delivery to an endpoint that did not consent.` + }) + }; + } + return { + check: eventsCheck(id, description, 'WARNING', { + errorMessage: `The subscribe succeeded although the callback echoed the wrong nonce. Nothing was delivered to it, so the endpoint is safe, but the subscriber was never told: the document returns ${EVENTS_CALLBACK_ENDPOINT_ERROR} "challenge_failed" synchronously from events/subscribe.` + }) + }; + } finally { + await release(url); + } + } + + /** A callback that redirects: the server must not follow it. */ + private async redirectChecks( + receiver: Receiver, + subscribe: ( + url: string + ) => Promise<{ id?: unknown } | { error: JsonRpcError }>, + release: (url: string) => Promise + ): Promise { + const from = `/redirect-${Date.now()}`; + const to = `/redirect-target-${Date.now()}`; + receiver.behave(from, { + kind: 'redirect', + to: this.callbackFor(receiver, to) + }); + const url = this.callbackFor(receiver, from); + const description = + 'Webhook delivery requests MUST NOT follow HTTP redirects, since a redirect can target an internal address that bypasses the blocklist.'; + + const sub = await subscribe(url); + if ('error' in sub) { + return [ + untestableCheck( + 'sep-3415-ssrf-no-redirects', + 'sep-3415-ssrf-no-redirects', + description, + `The redirecting callback could not be subscribed (${sub.error.code} ${sub.error.message}).`, + [EVENTS_SPEC_REF] + ) + ]; + } + try { + const redirected = await receiver.waitFor( + from, + (d) => d.respondedStatus === 302, + DELIVERY_WAIT_MS + ); + if (!redirected) { + return [ + untestableCheck( + 'sep-3415-ssrf-no-redirects', + 'sep-3415-ssrf-no-redirects', + description, + `Nothing was delivered to the redirecting callback within ${DELIVERY_WAIT_MS}ms, so no redirect was offered.`, + [EVENTS_SPEC_REF] + ) + ]; + } + // Give a server that does follow redirects time to arrive at the target. + const followed = await receiver.waitFor( + to, + () => true, + Math.min(3000, SETTLE_MS) + ); + return [ + followed + ? eventsCheck('sep-3415-ssrf-no-redirects', description, 'FAILURE', { + errorMessage: `The server followed a 302 from ${from} to ${to}. A redirect can point at an internal address, which is exactly what the delivery-time IP check is meant to stop.` + }) + : eventsCheck('sep-3415-ssrf-no-redirects', description, 'SUCCESS', { + details: { offered: from, notFollowed: to } + }) + ]; + } finally { + await release(url); + } + } + + /** Retries: fresh signatures, a bound, and the two non-retryable statuses. */ + private async retryChecks( + receiver: Receiver, + subscribe: ( + url: string + ) => Promise<{ id?: unknown } | { error: JsonRpcError }>, + release: (url: string) => Promise, + periodic: boolean + ): Promise { + const out: ConformanceCheck[] = []; + const flaky = `/flaky-${Date.now()}`; + receiver.behave(flaky, { + kind: 'fail-then-accept', + failures: 2, + status: 503 + }); + const flakyUrl = this.callbackFor(receiver, flaky); + + const sub = await subscribe(flakyUrl); + if ('error' in sub) { + out.push( + ...untestableAll( + [ + 'sep-3415-delivery-retry-regenerates-signature', + 'sep-3415-delivery-retries-bounded' + ], + `The retry probe could not be subscribed (${sub.error.code} ${sub.error.message}).` + ) + ); + } else { + try { + await receiver.waitFor( + flaky, + (d) => !isVerificationEnvelope(d), + DELIVERY_WAIT_MS + ); + // Let the retries play out. + await new Promise((resolve) => setTimeout(resolve, SETTLE_MS)); + const attempts = receiver.on(flaky); + const byId = new Map(); + for (const a of attempts) { + const id = a.headers['webhook-id'] ?? ''; + byId.set(id, [...(byId.get(id) ?? []), a]); + } + const retried = [...byId.values()].find((group) => group.length > 1); + + if (!retried) { + out.push( + ...untestableAll( + [ + 'sep-3415-delivery-retry-regenerates-signature', + 'sep-3415-delivery-retries-bounded' + ], + `A callback answering 503 received ${attempts.length} attempt(s), none of them a retry of the same \`webhook-id\`, so retry behaviour could not be observed.`, + 'WARNING' + ) + ); + } else { + const stamps = retried.map( + (a) => a.headers['webhook-timestamp'] ?? '' + ); + // Freshness only: whether the signature verifies at all belongs to + // sep-3415-delivery-signature-formula, and folding the two together + // reports a server with a wrong formula as reusing timestamps it + // plainly did not reuse. + // + // webhook-timestamp is whole seconds, so two attempts inside one + // second share a stamp whether or not it was regenerated. Only a + // stamp that stays put across attempts a second or more apart is + // provably reused. + const pairs = retried + .slice(1) + .map((a, i) => ({ prev: retried[i], next: a })); + const spaced = pairs.filter( + ({ prev, next }) => next.atMs - prev.atMs >= 1000 + ); + const reused = spaced.filter( + ({ prev, next }) => + (prev.headers['webhook-timestamp'] ?? '') === + (next.headers['webhook-timestamp'] ?? '') + ); + const description = + "Each retry attempt MUST regenerate the timestamp and signature so retries are not rejected by the receiver's freshness window."; + if (reused.length > 0) { + out.push( + eventsCheck( + 'sep-3415-delivery-retry-regenerates-signature', + description, + 'FAILURE', + { + errorMessage: `Retries of the same \`webhook-id\` reused a timestamp or signature (timestamps ${stamps.join(', ')}, the repeat at least a second apart), so a receiver enforcing the 5-minute freshness window would reject them.` + } + ) + ); + } else if ( + spaced.length > 0 || + new Set(stamps).size === stamps.length + ) { + out.push( + eventsCheck( + 'sep-3415-delivery-retry-regenerates-signature', + description, + 'SUCCESS', + { details: { attempts: retried.length, stamps } } + ) + ); + } else { + out.push( + eventsCheck( + 'sep-3415-delivery-retry-regenerates-signature', + description, + 'WARNING', + { + errorMessage: `Every retry arrived within a second of the one before (timestamps ${stamps.join(', ')}), and webhook-timestamp is in whole seconds, so a regenerated stamp and a reused one look the same.`, + details: { attempts: retried.length, stamps } + } + ) + ); + } + out.push( + retried.length <= 6 + ? eventsCheck( + 'sep-3415-delivery-retries-bounded', + 'Retries are bounded: servers SHOULD cap both the attempt count and the elapsed retry window.', + 'SUCCESS', + { details: { attempts: retried.length } } + ) + : eventsCheck( + 'sep-3415-delivery-retries-bounded', + 'Retries are bounded: servers SHOULD cap both the attempt count and the elapsed retry window (for example, 3–5 attempts over no more than 10–15 minutes).', + 'WARNING', + { + errorMessage: `One event was attempted ${retried.length} times within the observation window, past the 3–5 the document suggests.` + } + ) + ); + } + } finally { + await release(flakyUrl); + } + } + + // 410 Gone and 413 Payload Too Large are both non-retryable. The 410 probe + // also grades that the subscription outlives it, and the 413 probe, whose + // path keeps failing so nothing clears it, grades `lastError`. + let lastErrorGraded = false; + for (const [id, kind, status, description] of [ + [ + 'sep-3415-delivery-410-non-retryable', + 'gone' as const, + 410, + 'The server MUST treat `410` as a non-retryable failure for that delivery, without affecting the subscription itself.' + ], + [ + 'sep-3415-delivery-413-non-retryable', + 'too-large' as const, + 413, + 'Receivers and intermediaries MAY reject larger bodies with `413 Payload Too Large`; servers MUST treat `413` as a non-retryable failure for that event.' + ] + ] as const) { + const path = `/${kind}-${Date.now()}`; + receiver.behave(path, { kind }); + const url = this.callbackFor(receiver, path); + const probe = await subscribe(url); + if ('error' in probe) { + out.push( + untestableCheck( + id, + id, + description, + `The ${status} probe could not be subscribed (${probe.error.code} ${probe.error.message}).`, + [EVENTS_SPEC_REF] + ) + ); + continue; + } + try { + // The challenge on this path is answered 200 and never retried, so + // grading it would pass any server; the probe is about the event. + const first = await receiver.waitFor( + path, + (d) => !isVerificationEnvelope(d), + DELIVERY_WAIT_MS + ); + if (!first) { + out.push( + untestableCheck( + id, + id, + description, + `Nothing was delivered to the ${status} probe within ${DELIVERY_WAIT_MS}ms.`, + [EVENTS_SPEC_REF] + ) + ); + continue; + } + await new Promise((resolve) => setTimeout(resolve, SETTLE_MS)); + const repeats = receiver + .on(path) + .filter( + (d) => d.headers['webhook-id'] === first.headers['webhook-id'] + ); + if (repeats.length > 1) { + out.push( + eventsCheck(id, description, 'FAILURE', { + errorMessage: `The same \`webhook-id\` was delivered ${repeats.length} times after a ${status}, which the document defines as non-retryable.` + }) + ); + } else if (status === 410) { + out.push( + await this.goneSurvivalCheck(receiver, path, first, periodic) + ); + } else { + out.push( + eventsCheck(id, description, 'SUCCESS', { + details: { attempts: repeats.length } + }) + ); + } + if (status === 413) { + // A refresh on the same key; the 413 is the most recent outcome, and + // the path still answers 413, so nothing has cleared it. + const refreshed = await subscribe(url); + out.push(lastErrorCheck(refreshed)); + lastErrorGraded = true; + } + } finally { + await release(url); + } + } + if (!lastErrorGraded) { + out.push( + untestableCheck( + LAST_ERROR_ID, + LAST_ERROR_ID, + LAST_ERROR_DESCRIPTION, + 'The 413 probe delivered nothing, so there was no failure for `lastError` to report.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + } + + return out; + } + + /** + * After a 410, later events still reach the callback. Silence only means + * something when the source is known to fire more than once. + */ + private async goneSurvivalCheck( + receiver: Receiver, + path: string, + first: ReceivedDelivery, + periodic: boolean + ): Promise { + const id = 'sep-3415-delivery-410-non-retryable'; + const description = + 'The server MUST treat `410` as a non-retryable failure for that delivery, without affecting the subscription itself.'; + receiver.behave(path, { kind: 'accept' }); + const later = await receiver.waitFor( + path, + (d) => + isEvent(d) && d.headers['webhook-id'] !== first.headers['webhook-id'], + periodic ? DELIVERY_WAIT_MS : SETTLE_MS + ); + if (later) { + return eventsCheck(id, description, 'SUCCESS', { + details: { retried: false, laterDelivery: later.headers['webhook-id'] } + }); + } + if (periodic) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `After one \`410\`, no further event reached the callback within ${DELIVERY_WAIT_MS}ms, though the main subscription received several. A \`410\` is non-retryable for that delivery only, and must not suspend or drop the subscription.` + }); + } + return untestableCheck( + id, + id, + description, + 'The 410 was not retried, but the event source fired only once in this run, so whether the subscription outlived the 410 could not be observed.', + [EVENTS_SPEC_REF], + 'WARNING' + ); + } +} + +const LAST_ERROR_ID = 'sep-3415-delivery-status-last-error-category'; +const LAST_ERROR_DESCRIPTION = + '`lastError` MUST be a server-generated category string, one of `connection_refused`, `timeout`, `tls_error`, `http_4xx`, `http_5xx` or `challenge_failed`. It MUST NOT include raw response bodies, headers or status lines from the endpoint.'; + +/** `lastError` off a refresh that followed a failed delivery. */ +function lastErrorCheck( + refreshed: object | { error: JsonRpcError } +): ConformanceCheck { + const status = + 'error' in refreshed + ? undefined + : (refreshed as { deliveryStatus?: unknown }).deliveryStatus; + const lastError = + status !== null && typeof status === 'object' + ? (status as { lastError?: unknown }).lastError + : undefined; + if (lastError === undefined || lastError === null) { + return untestableCheck( + LAST_ERROR_ID, + LAST_ERROR_ID, + LAST_ERROR_DESCRIPTION, + 'error' in refreshed + ? `The refresh after the 413 failed (${refreshed.error.code} ${refreshed.error.message}).` + : '`deliveryStatus` is OPTIONAL, and the refresh after a failed delivery carried no `lastError`.', + [EVENTS_SPEC_REF], + 'WARNING' + ); + } + return typeof lastError === 'string' && + LAST_ERROR_CATEGORIES.includes(lastError) + ? eventsCheck(LAST_ERROR_ID, LAST_ERROR_DESCRIPTION, 'SUCCESS', { + details: { lastError } + }) + : eventsCheck(LAST_ERROR_ID, LAST_ERROR_DESCRIPTION, 'FAILURE', { + errorMessage: `\`lastError\` is ${describeValue(lastError)}, not one of ${LAST_ERROR_CATEGORIES.join(', ')}. Anything else can carry endpoint detail back to the subscriber.` + }); +} + +/** An event delivery, as opposed to a verification or control envelope. */ +function isEvent(d: ReceivedDelivery): boolean { + return !isVerificationEnvelope(d) && typeof d.json?.type !== 'string'; +} + +function isVerificationEnvelope(d: ReceivedDelivery): boolean { + return ( + d.json?.type === 'verification' || typeof d.json?.challenge === 'string' + ); +} + +/** Keep the first check emitted per id, so a fallback path cannot double-report. */ +/** + * What became of asking for each control envelope: no control, a control that + * declined, or a control that acknowledged (whether or not anything arrived). + */ +type SignalOutcome = 'absent' | 'refused' | 'sent'; +type SignalledEnvelopes = { gap: SignalOutcome; terminate: SignalOutcome }; + +function dedupe(checks: ConformanceCheck[]): ConformanceCheck[] { + const seen = new Set(); + return checks.filter((c) => { + if (seen.has(c.id)) return false; + seen.add(c.id); + return true; + }); +} diff --git a/src/scenarios/server/events/webhook.ts b/src/scenarios/server/events/webhook.ts new file mode 100644 index 00000000..e22aa491 --- /dev/null +++ b/src/scenarios/server/events/webhook.ts @@ -0,0 +1,1557 @@ +/** + * MCP Events: webhook subscription management — `events/subscribe`, + * TTL negotiation, and `events/unsubscribe`. + * + * Scored against the merged design sketch on `main` of + * modelcontextprotocol/experimental-ext-triggers-events. Each check's verbatim + * excerpt lives beside its id in src/seps/sep-3415.yaml. + * + * This scenario deliberately stops at the subscription surface. Everything + * about what the server then POSTs — signatures, headers, the verification + * handshake, control envelopes — belongs to events-webhook-delivery, which + * needs a callback URL the server can reach. Here the callback is a URL that + * exists only to be a distinct key, so a server may well suspend delivery to + * it; that is not this scenario's business. + * + * Rows the principal owns are the awkward ones. The subscription key is + * `(principal, delivery.url, name, arguments)`, and a run holds exactly one + * principal: whatever credential the harness was pointed at, or none. So the + * three components the harness can vary are graded, and the principal + * component is reported untestable with the reason named, rather than passing + * a cross-tenant isolation row that was never exercised across two tenants. + * + * Every subscription this scenario creates is unsubscribed before it returns. + * Against a public server a leaked subscription is a server holding state for + * a callback nobody reads, so cleanup is part of the scenario rather than an + * afterthought. + */ + +import { ClientScenario, ConformanceCheck } from '../../../types'; +import type { Connection, RunContext } from '../../../connection'; +import { JsonRpcError } from '../../../connection'; +import { untestableCheck } from '../../untestable'; +import { + EVENTS_EXTENSION_ID, + EVENTS_CONTROL_SUBSCRIBE_AS, + EVENTS_CONTROL_SUBSCRIPTION_EXISTS, + EVENTS_CONTROL_RESTART, + EVENTS_CONTROL_GENERATION, + EVENTS_CONTROL_SUBSCRIPTION_STATE, + hasControl, + askControl, + extensionsOf, + EVENTS_NOT_FOUND, + EVENTS_SPEC_REF, + EVENTS_SUBSCRIBE_METHOD, + EVENTS_UNSUBSCRIBE_METHOD, + resultTypeCheck, + RESULT_TYPE_ID, + type ObservedResult, + EVENTS_UNSUPPORTED, + JSONRPC_INVALID_PARAMS, + JSONRPC_METHOD_NOT_FOUND, + describeValue, + descriptorName, + eventsCheck, + eventsListAll, + firstSupporting, + isObject, + isValidCursor, + minimalArguments, + type EventDescriptor +} from './helpers'; + +/** A callback URL that is syntactically valid and points nowhere in use. */ +const CALLBACK_BASE = 'https://conformance.invalid/mcp-events'; +const DURABILITY_IDS = [ + 'sep-3415-ttl-long-grant-retained', + 'sep-3415-ttl-no-expiry-persisted', + 'sep-3415-ttl-no-expiry-gc-terminated' +] as const; +/** How long a restart may take to show up in the generation control. */ +const restartWaitMs = (): number => + Number(process.env.EVENTS_RESTART_WAIT_MS ?? 5000); +/** How long to watch a failing no-expiry subscription for a GC drop. */ +const gcWaitMs = (): number => Number(process.env.EVENTS_GC_WAIT_MS ?? 30000); + +/** A suggestion the server can plausibly grant, for the TTL rows. */ +const TTL_SUGGESTION_MS = 3600_000; + +const SUBSCRIBE_IDS = [ + 'sep-3415-subscribe-webhook-only', + 'sep-3415-subscribe-secret-required', + 'sep-3415-subscribe-secret-format', + 'sep-3415-subscribe-secret-rejected', + 'sep-3415-subscribe-url-https-required', + 'sep-3415-subscribe-url-non-https-rejected', + 'sep-3415-subscribe-auth-required', + 'sep-3415-subscribe-key-composition', + 'sep-3415-subscribe-key-immutable', + 'sep-3415-subscribe-idempotent-upsert', + 'sep-3415-subscribe-id-derived', + 'sep-3415-subscribe-id-not-an-input', + 'sep-3415-subscribe-refresh-replaces-secret', + 'sep-3415-subscribe-refresh-reactivates', + 'sep-3415-subscribe-response-cursor', + 'sep-3415-subscribe-response-truncated', + 'sep-3415-subscribe-refresh-before-present', + 'sep-3415-subscribe-cross-tenant-isolation' +] as const; + +const TTL_IDS = [ + 'sep-3415-ttl-refresh-before-lte-suggestion', + 'sep-3415-ttl-no-rejection-path', + 'sep-3415-ttl-null-only-when-requested', + 'sep-3415-ttl-omitted-means-default', + 'sep-3415-ttl-long-grant-retained', + 'sep-3415-ttl-no-expiry-persisted', + 'sep-3415-ttl-no-expiry-gc-terminated' +] as const; + +const UNSUBSCRIBE_IDS = [ + 'sep-3415-unsubscribe-by-key', + 'sep-3415-unsubscribe-empty-result', + 'sep-3415-unsubscribe-unknown-not-found' +] as const; + +/** + * The error-table row this scenario claims. It lives outside the three groups + * above because those name their own probes, and it has to be in `ALL_IDS` or a + * server this scenario bails on early emits 26 rows where a gradeable one emits + * 27 — which reads as a shorter suite rather than a prerequisite that was + * missing. + */ +const ERROR_IDS = ['sep-3415-error-unsupported'] as const; + +const ALL_IDS = [ + ...SUBSCRIBE_IDS, + ...TTL_IDS, + ...UNSUBSCRIBE_IDS, + ...ERROR_IDS, + RESULT_TYPE_ID +]; + +interface SubscribeResult { + id?: unknown; + refreshBefore?: unknown; + cursor?: unknown; + truncated?: unknown; + deliveryStatus?: unknown; +} + +/** How a subscription-state answer reads in a failure message. */ +function stateWord(state: string | undefined): string { + return state === undefined ? 'unreadable' : `reported \`${state}\``; +} + +/** A Standard Webhooks secret: `whsec_` plus base64 of 32 random bytes. */ +function freshSecret(): string { + const bytes = new Uint8Array(32); + for (let i = 0; i < bytes.length; i++) { + bytes[i] = Math.floor(Math.random() * 256); + } + return `whsec_${Buffer.from(bytes).toString('base64')}`; +} + +function untestableAll( + ids: readonly string[], + reason: string, + severity: 'FAILURE' | 'WARNING' = 'FAILURE' +): ConformanceCheck[] { + return ids.map((id) => + untestableCheck(id, id, id, reason, [EVENTS_SPEC_REF], severity) + ); +} + +function skipAll(reason: string): ConformanceCheck[] { + return ALL_IDS.map((id) => + eventsCheck(id, id, 'SKIPPED', { errorMessage: reason }) + ); +} + +export class EventsWebhookScenario implements ClientScenario { + name = 'events-webhook'; + readonly source = { extensionId: EVENTS_EXTENSION_ID } as const; + description = `MCP Events: webhook subscription management. + +**Methods**: \`events/subscribe\`, \`events/unsubscribe\`, plus \`events/list\` to discover an event type advertising \`webhook\` delivery + +**Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-3415.yaml): + +- \`sep-3415-subscribe-secret-*\` — the \`whsec_\` Standard Webhooks secret is required, and a malformed one is rejected with \`InvalidParams\` +- \`sep-3415-subscribe-url-*\` — callback URLs must be https, and a non-https one is rejected +- \`sep-3415-subscribe-key-*\` / \`sep-3415-subscribe-id-*\` — the compound key, its immutability, the derived \`id\`, and that \`id\` is not an input +- \`sep-3415-subscribe-idempotent-upsert\` / \`sep-3415-subscribe-response-*\` — a repeat subscribe refreshes in place and returns \`cursor\` and \`truncated\` +- \`sep-3415-ttl-*\` — the grant is at or under the suggestion, there is no rejection path, and \`null\` comes back only when asked for +- \`sep-3415-unsubscribe-*\` — teardown by key, and \`-32023 NotFound\` for a key the server does not hold + +**Scope**: subscription management only. Signatures, the verification handshake and control envelopes belong to events-webhook-delivery, which needs a reachable callback. + +**The principal is one row's blind spot**: the key is \`(principal, delivery.url, name, arguments)\` and a run holds one principal, so cross-tenant isolation and the auth requirement report untestable rather than passing unexercised. + +**Cleanup**: every subscription created here is unsubscribed before the scenario returns.`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect(); + try { + const capabilities = await conn.discover(); + const caps = isObject(capabilities.capabilities) + ? capabilities.capabilities + : {}; + const declared = extensionsOf(caps)[EVENTS_EXTENSION_ID] !== undefined; + + const listed = await eventsListAll(conn); + if ('error' in listed) { + if (!declared && listed.error.code === JSONRPC_METHOD_NOT_FOUND) { + return skipAll( + 'Server does not declare the `events` capability and does not implement `events/list`; the extension is optional.' + ); + } + return untestableAll( + ALL_IDS, + `\`events/list\` failed (${listed.error.code} ${listed.error.message}), so no webhook-capable event type could be discovered. See the events-discovery scenario.` + ); + } + + const target = firstSupporting(listed.descriptors, 'webhook'); + const name = target ? descriptorName(target) : undefined; + if (!target || !name) { + return untestableAll( + ALL_IDS, + listed.descriptors.length === 0 + ? '`events/list` returned an empty catalog, so no webhook-capable event type could be exercised.' + : 'No event type advertises `webhook` delivery, so `events/subscribe` could not be exercised. Webhook is optional per event type.' + ); + } + + const args = minimalArguments(target); + if (args === undefined) { + return untestableAll( + ALL_IDS, + `Event type \`${name}\` declares required \`inputSchema\` properties the harness cannot satisfy from the schema, so no subscription could be created.` + ); + } + + const checks = await this.webhookChecks( + conn, + listed.descriptors, + name, + args + ); + // Last, because the restart ends this connection. + const durable = await this.durabilityChecks(ctx, conn, name, args); + return checks.map((c) => durable.get(c.id) ?? c); + } finally { + await conn.close().catch(() => undefined); + } + } + + /** + * The three TTL rows that need a restart. Only reachable through a fixture + * exposing the restart and subscription-state controls; without them the + * untestable rows from ttlChecks stand. + */ + private async durabilityChecks( + ctx: RunContext, + conn: Connection, + name: string, + args: Record + ): Promise> { + const out = new Map(); + const canRestart = await hasControl(conn, EVENTS_CONTROL_RESTART); + const canAsk = + (await hasControl(conn, EVENTS_CONTROL_SUBSCRIPTION_STATE)) && + (await hasControl(conn, EVENTS_CONTROL_GENERATION)); + if (!canRestart || !canAsk) return out; + + const stamp = Date.now(); + const subscribeWith = async ( + path: string, + ttlMs: number | null + ): Promise<{ id: string; url: string; grant: unknown } | undefined> => { + const url = `${CALLBACK_BASE}/${path}-${stamp}`; + try { + const result = await conn.request( + EVENTS_SUBSCRIBE_METHOD, + { + name, + arguments: args, + delivery: { mode: 'webhook', url, secret: freshSecret() }, + ttlMs + } + ); + return typeof result?.id === 'string' + ? { id: result.id, url, grant: result.refreshBefore } + : undefined; + } catch { + return undefined; + } + }; + + const noExpiry = await subscribeWith('ttl-null', null); + const long = await subscribeWith('ttl-long', 24 * 3600 * 1000); + const granted = noExpiry !== undefined && noExpiry.grant === null; + + const target = await askControl(conn, EVENTS_CONTROL_RESTART, {}); + // A stateful server ends the old session, so the connection has to be + // replaced; a stateless one keeps answering on the same connection, from + // the old build and then the new. The generation settles both. + let live: Connection = conn; + let opened: Connection | undefined; + const deadline = Date.now() + restartWaitMs(); + let restarted = false; + while (target !== undefined && Date.now() < deadline) { + const gen = await askControl(live, EVENTS_CONTROL_GENERATION, {}); + if (gen === target) { + restarted = true; + break; + } + if (gen === undefined) { + try { + await opened?.close().catch(() => undefined); + opened = await ctx.connect(); + await opened.discover(); + live = opened; + } catch { + // The new build may not be up yet; try again next tick. + } + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + if (!restarted) { + await opened?.close().catch(() => undefined); + for (const id of DURABILITY_IDS) { + out.set( + id, + untestableCheck( + id, + id, + id, + `The restart control answered ${describeValue(target)} but the generation control never reported it within ${restartWaitMs()}ms, so there is no evidence a restart happened.`, + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + } + return out; + } + + const fresh = live; + try { + const state = async (id: string): Promise => + askControl(fresh, EVENTS_CONTROL_SUBSCRIPTION_STATE, { id }); + + if (long) { + const s = await state(long.id); + out.set( + 'sep-3415-ttl-long-grant-retained', + eventsCheck( + 'sep-3415-ttl-long-grant-retained', + 'A server granting long or no-expiry TTLs MUST retain subscriptions for the lifetime it granted, including across restarts.', + s === 'active' || s === 'suspended' ? 'SUCCESS' : 'FAILURE', + s === 'active' || s === 'suspended' + ? { details: { grant: long.grant, afterRestart: s } } + : { + errorMessage: `A subscription granted until ${String(long.grant)} was ${stateWord(s)} after a restart.` + } + ) + ); + } + + if (!granted) { + for (const id of [ + 'sep-3415-ttl-no-expiry-persisted', + 'sep-3415-ttl-no-expiry-gc-terminated' + ]) { + out.set( + id, + untestableCheck( + id, + id, + id, + `\`ttlMs: null\` was answered with refreshBefore ${describeValue(noExpiry?.grant)}, not null, so the server did not grant a no-expiry subscription. That is allowed; it only means these rows have nothing to grade.`, + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + } + return out; + } + + const afterRestart = await state(noExpiry.id); + const persisted = + afterRestart === 'active' || afterRestart === 'suspended'; + out.set( + 'sep-3415-ttl-no-expiry-persisted', + eventsCheck( + 'sep-3415-ttl-no-expiry-persisted', + 'The server MUST persist no-expiry subscriptions across restarts, because a client that never refreshes will never detect (or repair) a silently dropped one.', + persisted ? 'SUCCESS' : 'FAILURE', + persisted + ? { details: { afterRestart } } + : { + errorMessage: `A no-expiry subscription was ${stateWord(afterRestart)} after a restart. A client that never refreshes has no way to find that out.` + } + ) + ); + + // The callback origin never resolves, so every delivery fails; a server + // that GCs no-expiry subscriptions should drop this one. + const gcDescription = + 'The server MAY drop a no-expiry subscription after sustained delivery failure (server-defined window), and SHOULD attempt a `terminated` envelope when it does.'; + let dropped = false; + const gcDeadline = Date.now() + gcWaitMs(); + while (persisted && Date.now() < gcDeadline) { + if ((await state(noExpiry.id)) === 'absent') { + dropped = true; + break; + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + out.set( + 'sep-3415-ttl-no-expiry-gc-terminated', + !persisted + ? untestableCheck( + 'sep-3415-ttl-no-expiry-gc-terminated', + 'sep-3415-ttl-no-expiry-gc-terminated', + gcDescription, + 'The no-expiry subscription was already gone after the restart, so there was nothing left for failure-based GC to drop.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + : dropped + ? eventsCheck( + 'sep-3415-ttl-no-expiry-gc-terminated', + gcDescription, + 'SUCCESS', + { + details: { + note: 'Dropped after sustained delivery failure. The terminated envelope goes to the placeholder callback origin, which the harness cannot receive at, so only the drop is observed.' + } + } + ) + : eventsCheck( + 'sep-3415-ttl-no-expiry-gc-terminated', + gcDescription, + 'WARNING', + { + errorMessage: `A no-expiry subscription whose every delivery failed was still registered after ${gcWaitMs()}ms. Dropping it is a MAY, so this is not a violation; a server with a long GC window reads the same way.` + } + ) + ); + + for (const sub of [long, noExpiry]) { + if (!sub) continue; + await fresh + .request(EVENTS_UNSUBSCRIBE_METHOD, { + name, + arguments: args, + delivery: { mode: 'webhook', url: sub.url } + }) + .catch(() => undefined); + } + return out; + } finally { + await opened?.close().catch(() => undefined); + } + } + + private async webhookChecks( + conn: Connection, + descriptors: EventDescriptor[], + name: string, + args: Record + ): Promise { + const checks: ConformanceCheck[] = []; + const created: Array<{ url: string; name: string }> = []; + const url = `${CALLBACK_BASE}/${Date.now()}`; + + // A server may cap concurrent subscriptions per principal — kitchen-sink + // allows two per event type — and a run that holds every probe's + // subscription open hits that cap and then grades the cap error instead of + // the rule it was probing. So each throwaway subscription is released as + // soon as its check is graded, and only the one the later checks build on + // is held. + const release = async (url: string, subName: string): Promise => { + try { + await conn.request(EVENTS_UNSUBSCRIBE_METHOD, { + name: subName, + arguments: args, + delivery: { mode: 'webhook', url } + }); + } catch { + // Never created, or already gone. + } + const at = created.findIndex((c) => c.url === url && c.name === subName); + if (at >= 0) created.splice(at, 1); + }; + + const subscribe = async ( + params: Record + ): Promise<{ result: SubscribeResult } | { error: JsonRpcError }> => { + try { + const result = await conn.request( + EVENTS_SUBSCRIBE_METHOD, + params + ); + const delivery = isObject(params.delivery) ? params.delivery : {}; + if (typeof delivery.url === 'string') { + created.push({ url: delivery.url, name: String(params.name) }); + } + return { result: result ?? {} }; + } catch (err) { + if (err instanceof JsonRpcError) return { error: err }; + throw err; + } + }; + + try { + // --- A subscription the rest of the scenario builds on ---------------- + const first = await subscribe({ + name, + arguments: args, + delivery: { mode: 'webhook', url, secret: freshSecret() }, + cursor: null, + ttlMs: TTL_SUGGESTION_MS + }); + + if ('error' in first) { + const err = first.error; + checks.push( + eventsCheck( + 'sep-3415-subscribe-webhook-only', + '`events/subscribe` is ONLY used for webhook delivery. Poll and push do not need it.', + err.code === JSONRPC_METHOD_NOT_FOUND ? 'FAILURE' : 'WARNING', + { + errorMessage: `Event type \`${name}\` advertises \`webhook\` delivery but a well-formed \`${EVENTS_SUBSCRIBE_METHOD}\` answered ${err.code} ${err.message}.`, + details: { code: err.code, message: err.message, data: err.data } + } + ) + ); + checks.push( + ...untestableAll( + ALL_IDS.filter((id) => id !== 'sep-3415-subscribe-webhook-only'), + `No subscription could be created for \`${name}\` (${err.code} ${err.message}), so the subscription surface could not be exercised.` + ) + ); + return dedupe(checks); + } + + checks.push( + eventsCheck( + 'sep-3415-subscribe-webhook-only', + '`events/subscribe` is ONLY used for webhook delivery. Poll and push do not need it.', + 'SUCCESS', + { details: { name } } + ) + ); + + checks.push(...this.responseChecks(first.result)); + checks.push(...this.ttlChecks(first.result)); + checks.push( + ...(await this.ttlNegotiationChecks( + subscribe, + release, + name, + args, + url + )) + ); + checks.push( + ...(await this.rejectionChecks( + subscribe, + release, + name, + args, + descriptors + )) + ); + checks.push( + ...(await this.identityChecks( + conn, + subscribe, + release, + first.result, + name, + args, + url + )) + ); + checks.push( + ...(await this.unsubscribeChecks(conn, first.result, name, args, url)) + ); + return dedupe(checks); + } finally { + // Leave nothing behind, including on a public server. + for (const sub of created) { + try { + await conn.request(EVENTS_UNSUBSCRIBE_METHOD, { + name: sub.name, + arguments: args, + delivery: { mode: 'webhook', url: sub.url } + }); + } catch { + // Already gone, or never created. Nothing to do. + } + } + } + } + + /** `cursor`, `truncated` and the presence of `refreshBefore` on the subscribe response. */ + private responseChecks(result: SubscribeResult): ConformanceCheck[] { + const out: ConformanceCheck[] = []; + // Presence only: whether a finite grant is well formed and within the + // suggestion is sep-3415-ttl-refresh-before-lte-suggestion's business. + const refreshDescription = + "`refreshBefore` (response, always present, nullable) is the server's authoritative grant: an ISO 8601 timestamp for when the subscription expires, or `null` for no expiry."; + out.push( + Object.prototype.hasOwnProperty.call(result, 'refreshBefore') + ? eventsCheck( + 'sep-3415-subscribe-refresh-before-present', + refreshDescription, + 'SUCCESS', + { details: { refreshBefore: result.refreshBefore } } + ) + : eventsCheck( + 'sep-3415-subscribe-refresh-before-present', + refreshDescription, + 'FAILURE', + { + errorMessage: + 'The subscribe response has no `refreshBefore` member. It is always present, and `null` is how a server grants no expiry, so an absent field leaves the client unable to tell a grant from an omission.' + } + ) + ); + out.push( + isValidCursor(result.cursor) + ? eventsCheck( + 'sep-3415-subscribe-response-cursor', + "The subscribe response carries `cursor`, a safe-to-persist watermark that advances the client's cursor even if no events arrive before next refresh.", + 'SUCCESS', + { details: { cursor: result.cursor } } + ) + : eventsCheck( + 'sep-3415-subscribe-response-cursor', + "The subscribe response carries `cursor`, a safe-to-persist watermark that advances the client's cursor even if no events arrive before next refresh.", + 'FAILURE', + { + errorMessage: `\`cursor\` is ${describeValue(result.cursor)}, expected a string or null.` + } + ) + ); + out.push( + result.truncated === undefined || typeof result.truncated === 'boolean' + ? eventsCheck( + 'sep-3415-subscribe-response-truncated', + 'The subscribe response carries `truncated`, true if delivery started later than the supplied cursor.', + 'SUCCESS', + { details: { truncated: result.truncated ?? false } } + ) + : eventsCheck( + 'sep-3415-subscribe-response-truncated', + 'The subscribe response carries `truncated`, true if delivery started later than the supplied cursor.', + 'FAILURE', + { + errorMessage: `\`truncated\` is ${describeValue(result.truncated)}, expected a boolean.` + } + ) + ); + return out; + } + + /** The grant on the subscription the scenario opened with a suggestion. */ + private ttlChecks(result: SubscribeResult): ConformanceCheck[] { + const out: ConformanceCheck[] = []; + const grant = result.refreshBefore; + const description = + '`refreshBefore` (response) is the grant. It SHOULD be less than or equal to the suggestion.'; + + if (grant === null) { + out.push( + eventsCheck( + 'sep-3415-ttl-null-only-when-requested', + 'A server MUST NOT return `null` unless the client suggested `ttlMs: null` — no expiry exceeds every finite suggestion.', + 'FAILURE', + { + errorMessage: `The subscribe carried \`ttlMs: ${TTL_SUGGESTION_MS}\` and the server granted \`refreshBefore: null\`, which is no expiry.` + } + ) + ); + out.push( + untestableCheck( + 'sep-3415-ttl-refresh-before-lte-suggestion', + 'sep-3415-ttl-refresh-before-lte-suggestion', + description, + 'The server granted no expiry, so there is no finite grant to compare against the suggestion.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + return out; + } + + out.push( + eventsCheck( + 'sep-3415-ttl-null-only-when-requested', + 'A server MUST NOT return `null` unless the client suggested `ttlMs: null` — no expiry exceeds every finite suggestion.', + 'SUCCESS', + { details: { refreshBefore: grant } } + ) + ); + + const grantedAt = typeof grant === 'string' ? Date.parse(grant) : NaN; + if (!Number.isFinite(grantedAt)) { + out.push( + eventsCheck( + 'sep-3415-ttl-refresh-before-lte-suggestion', + description, + 'FAILURE', + { + errorMessage: `\`refreshBefore\` is ${describeValue(grant)}, expected an ISO 8601 timestamp or null.` + } + ) + ); + return out; + } + + // A minute of slack: the suggestion is measured from when the harness + // sent the request, the grant from when the server handled it. + const ceiling = Date.now() + TTL_SUGGESTION_MS + 60_000; + out.push( + grantedAt <= ceiling + ? eventsCheck( + 'sep-3415-ttl-refresh-before-lte-suggestion', + description, + 'SUCCESS', + { + details: { refreshBefore: grant, suggestedMs: TTL_SUGGESTION_MS } + } + ) + : eventsCheck( + 'sep-3415-ttl-refresh-before-lte-suggestion', + description, + 'WARNING', + { + errorMessage: `Granted \`refreshBefore\` is ${grant}, about ${Math.round((grantedAt - Date.now()) / 1000)}s out, past the suggested ${TTL_SUGGESTION_MS / 1000}s.` + } + ) + ); + return out; + } + + /** Omitted, absurd and no-expiry TTL suggestions. */ + private async ttlNegotiationChecks( + subscribe: ( + p: Record + ) => Promise<{ result: SubscribeResult } | { error: JsonRpcError }>, + release: (url: string, name: string) => Promise, + name: string, + args: Record, + baseUrl: string + ): Promise { + const out: ConformanceCheck[] = []; + + // Omitting ttlMs means "server default", which is still a grant. + const omitted = await subscribe({ + name, + arguments: args, + delivery: { + mode: 'webhook', + url: `${baseUrl}-ttl-default`, + secret: freshSecret() + }, + cursor: null + }); + await release(`${baseUrl}-ttl-default`, name); + out.push( + 'error' in omitted + ? eventsCheck( + 'sep-3415-ttl-omitted-means-default', + 'Omitting `ttlMs` means "server default."', + 'FAILURE', + { + errorMessage: `A subscribe with no \`ttlMs\` answered ${omitted.error.code} ${omitted.error.message}; omitting it is defined as asking for the server default.` + } + ) + : omitted.result.refreshBefore === null + ? eventsCheck( + 'sep-3415-ttl-omitted-means-default', + 'Omitting `ttlMs` means "server default." An explicit `ttlMs: null` requests a subscription with no expiry.', + 'FAILURE', + { + errorMessage: + 'A subscribe with no `ttlMs` was granted `refreshBefore: null`, which only an explicit `ttlMs: null` may request.' + } + ) + : eventsCheck( + 'sep-3415-ttl-omitted-means-default', + 'Omitting `ttlMs` means "server default."', + 'SUCCESS', + { details: { refreshBefore: omitted.result.refreshBefore } } + ) + ); + + // Clamping is self-announcing in both directions, so neither an + // impractically short nor a very long suggestion is an error. + const short = await subscribe({ + name, + arguments: args, + delivery: { + mode: 'webhook', + url: `${baseUrl}-ttl-short`, + secret: freshSecret() + }, + cursor: null, + ttlMs: 1000 + }); + // Released before the next one is opened: a server may cap concurrent + // subscriptions per principal, and the primary subscription holds a slot. + await release(`${baseUrl}-ttl-short`, name); + const long = await subscribe({ + name, + arguments: args, + delivery: { + mode: 'webhook', + url: `${baseUrl}-ttl-long`, + secret: freshSecret() + }, + cursor: null, + ttlMs: 30 * 24 * 3600_000 + }); + await release(`${baseUrl}-ttl-long`, name); + const rejected = [ + ['1000ms', short], + ['30 days', long] + ].filter(([, r]) => 'error' in (r as object)) as Array< + [string, { error: JsonRpcError }] + >; + out.push( + rejected.length === 0 + ? eventsCheck( + 'sep-3415-ttl-no-rejection-path', + 'Clamping in either direction is self-announcing, so a clamped grant is not an error and there is no rejection path for TTL values.', + 'SUCCESS', + { + details: { + short: + 'error' in short ? undefined : short.result.refreshBefore, + long: 'error' in long ? undefined : long.result.refreshBefore + } + } + ) + : eventsCheck( + 'sep-3415-ttl-no-rejection-path', + 'Clamping in either direction is self-announcing, so a clamped grant is not an error and there is no rejection path for TTL values.', + 'FAILURE', + { + errorMessage: rejected + .map( + ([label, r]) => + `\`ttlMs\` of ${label} was rejected with ${r.error.code} ${r.error.message}` + ) + .join('; ') + } + ) + ); + + // The durability rows need a restart, which the harness cannot ask for. + out.push( + ...untestableAll( + [ + 'sep-3415-ttl-long-grant-retained', + 'sep-3415-ttl-no-expiry-persisted', + 'sep-3415-ttl-no-expiry-gc-terminated' + ], + 'Grading retention across a restart needs the server under test to be restarted mid-run, which the harness cannot do over the wire.', + 'WARNING' + ) + ); + + return out; + } + + /** What a malformed subscribe must be rejected with. */ + private async rejectionChecks( + subscribe: ( + p: Record + ) => Promise<{ result: SubscribeResult } | { error: JsonRpcError }>, + release: (url: string, name: string) => Promise, + name: string, + args: Record, + descriptors: EventDescriptor[] + ): Promise { + const out: ConformanceCheck[] = []; + + const expectInvalidParams = async ( + id: string, + description: string, + params: Record, + what: string + ): Promise => { + const res = await subscribe(params); + if (!('error' in res)) { + // A probe the server was supposed to reject but accepted has just + // taken a subscription slot, and holding it would make the next probe + // grade a cap error instead of its own rule. + const delivery = isObject(params.delivery) ? params.delivery : {}; + if (typeof delivery.url === 'string') { + await release(delivery.url, String(params.name)); + } + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `${what} was accepted; the document requires \`-32602 InvalidParams\`.`, + details: { result: res.result } + }); + } + return res.error.code === JSONRPC_INVALID_PARAMS + ? eventsCheck(id, description, 'SUCCESS', { + details: { code: res.error.code } + }) + : eventsCheck(id, description, 'WARNING', { + errorMessage: `${what} was rejected with ${res.error.code} ${res.error.message}, where the document names \`-32602 InvalidParams\`.`, + details: { error: res.error } + }); + }; + + const base = { + name, + arguments: args, + cursor: null + }; + + out.push( + await expectInvalidParams( + 'sep-3415-subscribe-secret-required', + '`delivery.secret` is REQUIRED. The client supplies the HMAC signing secret; the server never generates one.', + { + ...base, + delivery: { mode: 'webhook', url: `${CALLBACK_BASE}/no-secret` } + }, + 'A subscribe with no `delivery.secret`' + ) + ); + + out.push( + await expectInvalidParams( + 'sep-3415-subscribe-secret-format', + 'The value MUST be a Standard Webhooks symmetric secret: the literal prefix `whsec_` followed by base64 of 24–64 random bytes.', + { + ...base, + delivery: { + mode: 'webhook', + url: `${CALLBACK_BASE}/bad-prefix`, + secret: 'not-a-standard-webhooks-secret' + } + }, + 'A `delivery.secret` without the `whsec_` prefix' + ) + ); + + // Right prefix, too few bytes: 8 decoded, where the floor is 24. + out.push( + await expectInvalidParams( + 'sep-3415-subscribe-secret-rejected', + 'Servers MUST reject a `delivery.secret` that is not `whsec_` followed by base64 decoding to 24–64 bytes with `InvalidParams`.', + { + ...base, + delivery: { + mode: 'webhook', + url: `${CALLBACK_BASE}/short-secret`, + secret: `whsec_${Buffer.from(new Uint8Array(8)).toString('base64')}` + } + }, + 'A `whsec_` secret decoding to 8 bytes, under the 24-byte floor,' + ) + ); + + const nonHttps = await expectInvalidParams( + 'sep-3415-subscribe-url-non-https-rejected', + 'Servers MUST reject `events/subscribe` with a non-`https` `delivery.url` (`-32602 InvalidParams`).', + { + ...base, + delivery: { + mode: 'webhook', + url: 'http://conformance.invalid/insecure', + secret: freshSecret() + } + }, + 'A subscribe with an `http://` `delivery.url`' + ); + out.push(nonHttps); + // The requirement and its enforcement are one probe; report both so the + // manifest does not carry an untested row for the rule itself. + out.push( + eventsCheck( + 'sep-3415-subscribe-url-https-required', + 'Callback URLs MUST use `https://`.', + nonHttps.status, + { + errorMessage: nonHttps.errorMessage, + details: { gradedBy: 'sep-3415-subscribe-url-non-https-rejected' } + } + ) + ); + + // A type that does not offer webhook, when the catalog has one. + const nonWebhook = descriptors.find((d) => { + const modes = Array.isArray(d.delivery) ? d.delivery : []; + return modes.length > 0 && !modes.includes('webhook'); + }); + const nonWebhookName = nonWebhook ? descriptorName(nonWebhook) : undefined; + if (!nonWebhookName) { + out.push( + untestableCheck( + 'sep-3415-error-unsupported', + 'sep-3415-error-unsupported', + '`-32026 Unsupported` — the request is well-formed but a requested capability or option is not supported here.', + 'Every event type the server offers advertises `webhook` delivery, so there is no type to probe the unsupported-mode path with.', + [EVENTS_SPEC_REF] + ) + ); + } else { + const res = await subscribe({ + name: nonWebhookName, + arguments: {}, + delivery: { + mode: 'webhook', + url: `${CALLBACK_BASE}/unsupported`, + secret: freshSecret() + }, + cursor: null + }); + out.push( + !('error' in res) + ? eventsCheck( + 'sep-3415-error-unsupported', + '`-32026 Unsupported` — the request is well-formed but a requested capability or option is not supported here.', + 'FAILURE', + { + errorMessage: `Event type \`${nonWebhookName}\` does not advertise \`webhook\` delivery but \`${EVENTS_SUBSCRIBE_METHOD}\` returned a subscription.` + } + ) + : res.error.code === EVENTS_UNSUPPORTED + ? eventsCheck( + 'sep-3415-error-unsupported', + '`-32026 Unsupported` — the request is well-formed but a requested capability or option is not supported here.', + 'SUCCESS', + { details: { code: res.error.code } } + ) + : eventsCheck( + 'sep-3415-error-unsupported', + '`-32026 Unsupported` — the request is well-formed but a requested capability or option is not supported here.', + 'WARNING', + { + errorMessage: `Subscribing to \`${nonWebhookName}\`, which offers ${JSON.stringify(nonWebhook?.delivery)}, answered ${res.error.code} ${res.error.message} rather than \`-32026 Unsupported\`.` + } + ) + ); + } + + // The principal half of the surface. One run, one credential. + out.push( + untestableCheck( + 'sep-3415-subscribe-auth-required', + 'sep-3415-subscribe-auth-required', + '`events/subscribe` and `events/unsubscribe` MUST be called with an authenticated principal; servers MUST reject calls without one with `-32024 Forbidden`.', + 'The harness sends whatever credential it was pointed at, on every request, and has no way to make the same call as an anonymous caller. Grading this needs a runner that can drop its own auth for one probe.', + [EVENTS_SPEC_REF] + ) + ); + + return out; + } + + /** The compound key, the derived id, and what a refresh does. */ + /** + * Grade sep-3415-subscribe-cross-tenant-isolation, which asserts that two + * tenants subscribing to the same `(name, arguments)` with the same callback + * get distinct subscriptions, because the principal is part of the key. + * + * A run authenticates as one principal for its lifetime, so the harness can + * only ever supply one side of that comparison. The other side comes from a + * fixture control that registers on another principal's behalf, using the + * same identity function the subscribe handler uses. + * + * Two things are checked, and the second is the one that bites. Distinct ids + * show the principal reached the key at all. A surviving subscription after + * the other tenant unsubscribes shows the two are genuinely independent, + * which is what a tenant would actually notice. + */ + private async crossTenantChecks( + conn: Connection, + name: string, + subscribe: ( + params: Record + ) => Promise<{ result: SubscribeResult } | { error: JsonRpcError }> + ): Promise { + const id = 'sep-3415-subscribe-cross-tenant-isolation'; + const description = + 'Because the key includes `principal` and `delivery.url`, two distinct tenants subscribing to the same `(name, arguments)` get distinct subscriptions.'; + + const canAct = await hasControl(conn, EVENTS_CONTROL_SUBSCRIBE_AS); + const canAsk = await hasControl(conn, EVENTS_CONTROL_SUBSCRIPTION_EXISTS); + if (!canAct || !canAsk) { + return [ + untestableCheck( + id, + id, + description, + `A run holds one principal, so the two-tenant case cannot be constructed. Needs a fixture exposing the \`${EVENTS_CONTROL_SUBSCRIBE_AS}\` and \`${EVENTS_CONTROL_SUBSCRIPTION_EXISTS}\` controls. The \`delivery.url\` half of the same rule is graded by sep-3415-subscribe-key-composition.`, + [EVENTS_SPEC_REF] + ) + ]; + } + + // Same event, same callback. The principal is the only thing that differs, + // which is what makes a shared id mean what it means. + const url = `${CALLBACK_BASE}/cross-tenant`; + const other = { principal: 'conformance-tenant-b', name, url }; + + const otherId = await askControl(conn, EVENTS_CONTROL_SUBSCRIBE_AS, other); + if (!otherId) { + return [ + untestableCheck( + id, + id, + description, + `The \`${EVENTS_CONTROL_SUBSCRIBE_AS}\` control did not return a subscription id, so the second tenant could not be established.`, + [EVENTS_SPEC_REF] + ) + ]; + } + + const mine = await subscribe({ + name, + arguments: {}, + delivery: { mode: 'webhook', url, secret: freshSecret() } + }); + if ('error' in mine) { + return [ + eventsCheck(id, description, 'FAILURE', { + errorMessage: `Subscribing as this run's own principal to the same event and callback failed with ${mine.error.code} ${mine.error.message}, so the two subscriptions could not be compared.` + }) + ]; + } + + const myId = mine.result.id; + if (myId === otherId) { + return [ + eventsCheck(id, description, 'FAILURE', { + errorMessage: `Two principals subscribing to \`${name}\` with the same callback received the same subscription id (${myId}), so the principal is not part of the key. One tenant can address, refresh or cancel another's subscription.`, + details: { id: myId } + }) + ]; + } + + // By key: the derived id is not accepted as input, so unsubscribing by id + // removes nothing on a conformant server, and the other tenant would + // "survive" whatever the server does. + try { + await conn.request(EVENTS_UNSUBSCRIBE_METHOD, { + name, + arguments: {}, + delivery: { mode: 'webhook', url } + }); + } catch (err) { + const why = + err instanceof JsonRpcError + ? `${err.code} ${err.message}` + : String(err); + return [ + untestableCheck( + id, + id, + description, + `Unsubscribing this run's own subscription by key failed (${why}), so whether the other tenant's subscription survives it says nothing.`, + [EVENTS_SPEC_REF] + ) + ]; + } + const survived = await askControl( + conn, + EVENTS_CONTROL_SUBSCRIPTION_EXISTS, + other + ); + + return [ + survived === 'true' + ? eventsCheck(id, description, 'SUCCESS', { + details: { mine: myId, other: otherId } + }) + : eventsCheck(id, description, 'FAILURE', { + errorMessage: `The two tenants received distinct ids, but unsubscribing \`${myId}\` also removed the other tenant's subscription, so they are not independent.`, + details: { mine: myId, other: otherId } + }) + ]; + } + + private async identityChecks( + conn: Connection, + subscribe: ( + p: Record + ) => Promise<{ result: SubscribeResult } | { error: JsonRpcError }>, + release: (url: string, name: string) => Promise, + first: SubscribeResult, + name: string, + args: Record, + url: string + ): Promise { + const out: ConformanceCheck[] = []; + const firstId = first.id; + + out.push( + typeof firstId === 'string' && firstId.length > 0 + ? eventsCheck( + 'sep-3415-subscribe-id-derived', + 'The server computes a deterministic `id` over the key and returns it in the subscribe response. It is stable across refreshes and server restarts.', + 'SUCCESS', + { details: { id: firstId } } + ) + : eventsCheck( + 'sep-3415-subscribe-id-derived', + 'The server computes a deterministic `id` over the key and returns it in the subscribe response.', + 'FAILURE', + { + errorMessage: `The subscribe response carried \`id\` ${describeValue(firstId)}, expected a string.` + } + ) + ); + + // Same key again: same id, and the TTL is re-granted in place. + const again = await subscribe({ + name, + arguments: args, + delivery: { mode: 'webhook', url, secret: freshSecret() }, + cursor: null, + ttlMs: TTL_SUGGESTION_MS + }); + if ('error' in again) { + out.push( + eventsCheck( + 'sep-3415-subscribe-idempotent-upsert', + '`events/subscribe` is idempotent — calling it again with the same subscription key refreshes the TTL and updates mutable fields.', + 'FAILURE', + { + errorMessage: `A second subscribe with the same key answered ${again.error.code} ${again.error.message}; the call is defined as an idempotent upsert.` + } + ) + ); + out.push( + ...untestableAll( + [ + 'sep-3415-subscribe-key-composition', + 'sep-3415-subscribe-key-immutable', + 'sep-3415-subscribe-refresh-replaces-secret' + ], + 'The repeat subscribe failed, so nothing about key identity could be compared.' + ) + ); + } else { + out.push( + again.result.id === firstId + ? eventsCheck( + 'sep-3415-subscribe-idempotent-upsert', + '`events/subscribe` is idempotent — calling it again with the same subscription key refreshes the TTL and updates mutable fields in place.', + 'SUCCESS', + { details: { id: firstId } } + ) + : eventsCheck( + 'sep-3415-subscribe-idempotent-upsert', + '`events/subscribe` is idempotent — calling it again with the same subscription key refreshes the TTL and updates mutable fields in place.', + 'FAILURE', + { + errorMessage: `The same key produced a different \`id\` (${describeValue(firstId)} then ${describeValue(again.result.id)}), so the second call created a second subscription.` + } + ) + ); + + // A refresh carrying a new secret is accepted; whether the next delivery + // is signed with it belongs to events-webhook-delivery. + out.push( + untestableCheck( + 'sep-3415-subscribe-refresh-replaces-secret', + 'sep-3415-subscribe-refresh-replaces-secret', + 'On an idempotent subscribe against an existing key, `delivery.secret` is replaced.', + 'The refresh carrying a new secret was accepted, but confirming the replacement needs a delivery signed with it, which events-webhook-delivery covers.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + } + + // Vary one component at a time: a different url, then different arguments. + const otherUrl = await subscribe({ + name, + arguments: args, + delivery: { mode: 'webhook', url: `${url}-other`, secret: freshSecret() }, + cursor: null + }); + await release(`${url}-other`, name); + const distinctIds = + !('error' in otherUrl) && + typeof otherUrl.result.id === 'string' && + otherUrl.result.id !== firstId; + out.push( + 'error' in otherUrl + ? untestableCheck( + 'sep-3415-subscribe-key-composition', + 'sep-3415-subscribe-key-composition', + 'The subscription key is `(principal, delivery.url, name, arguments)`.', + `A subscribe differing only in \`delivery.url\` answered ${otherUrl.error.code} ${otherUrl.error.message}, so the two keys could not be compared.`, + [EVENTS_SPEC_REF] + ) + : distinctIds + ? eventsCheck( + 'sep-3415-subscribe-key-composition', + 'The subscription key is `(principal, delivery.url, name, arguments)`; a call differing in any component addresses a different subscription.', + 'SUCCESS', + { + details: { + note: 'Graded on `delivery.url`; the `principal` component needs a second tenant.', + ids: [firstId, otherUrl.result.id] + } + } + ) + : eventsCheck( + 'sep-3415-subscribe-key-composition', + 'The subscription key is `(principal, delivery.url, name, arguments)`; a call differing in any component addresses a different subscription.', + 'FAILURE', + { + errorMessage: `Two subscriptions differing in \`delivery.url\` share the id ${describeValue(otherUrl.result.id)}, so the URL is not part of the key.` + } + ) + ); + out.push( + 'error' in otherUrl + ? untestableCheck( + 'sep-3415-subscribe-key-immutable', + 'sep-3415-subscribe-key-immutable', + "All four key components are immutable for the subscription's lifetime.", + 'The second subscription could not be created, so immutability could not be observed.', + [EVENTS_SPEC_REF] + ) + : eventsCheck( + 'sep-3415-subscribe-key-immutable', + "All four components are immutable for the subscription's lifetime: a subscribe call with a different value for any of them addresses a different subscription.", + distinctIds ? 'SUCCESS' : 'FAILURE', + { + errorMessage: distinctIds + ? undefined + : 'Changing `delivery.url` did not address a different subscription, so a key component was mutated in place.' + } + ) + ); + + // `id` is a routing handle, never an input. + const byId = await subscribe({ + name, + arguments: args, + id: firstId, + delivery: { mode: 'webhook', url: `${url}-by-id`, secret: freshSecret() }, + cursor: null + }); + await release(`${url}-by-id`, name); + out.push( + 'error' in byId + ? eventsCheck( + 'sep-3415-subscribe-id-not-an-input', + "A caller who learns another tenant's derived `id` gains nothing — `id` is not accepted as input to any method.", + 'SUCCESS', + { + details: { + note: 'The server rejected a subscribe carrying `id`.', + code: byId.error.code + } + } + ) + : byId.result.id === firstId + ? eventsCheck( + 'sep-3415-subscribe-id-not-an-input', + "A caller who learns another tenant's derived `id` gains nothing — `id` is not accepted as input to any method.", + 'FAILURE', + { + errorMessage: `A subscribe carrying \`id: ${String(firstId)}\` with a different \`delivery.url\` returned that same id, so \`id\` addressed the subscription instead of the key.` + } + ) + : eventsCheck( + 'sep-3415-subscribe-id-not-an-input', + "A caller who learns another tenant's derived `id` gains nothing — `id` is not accepted as input to any method.", + 'SUCCESS', + { + details: { + note: 'The supplied `id` was ignored; the key decided the subscription.', + ids: [firstId, byId.result.id] + } + } + ) + ); + + out.push(...(await this.crossTenantChecks(conn, name, subscribe))); + + out.push( + untestableCheck( + 'sep-3415-subscribe-refresh-reactivates', + 'sep-3415-subscribe-refresh-reactivates', + 'On an idempotent subscribe against an existing key, `active` is set to `true` and suspended delivery resumes.', + 'Suspension is observable only through the OPTIONAL `deliveryStatus` object, and reaching it needs sustained delivery failure against a callback the harness controls.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + + return out; + } + + /** Teardown by key, and what an unknown key answers. */ + private async unsubscribeChecks( + conn: Connection, + first: SubscribeResult, + name: string, + args: Record, + url: string + ): Promise { + const out: ConformanceCheck[] = []; + const unsubscribe = async ( + params: Record + ): Promise<{ result: unknown } | { error: JsonRpcError }> => { + try { + const result = await conn.request(EVENTS_UNSUBSCRIBE_METHOD, params); + return { result }; + } catch (err) { + if (err instanceof JsonRpcError) return { error: err }; + throw err; + } + }; + + const byKey = await unsubscribe({ + name, + arguments: args, + delivery: { mode: 'webhook', url } + }); + out.push( + 'error' in byKey + ? eventsCheck( + 'sep-3415-unsubscribe-by-key', + '`events/unsubscribe` is eager cleanup; the server looks the subscription up by the same compound key used for idempotent upsert on subscribe.', + 'FAILURE', + { + errorMessage: `Unsubscribing the subscription just created (id ${describeValue(first.id)}) by its key answered ${byKey.error.code} ${byKey.error.message}.` + } + ) + : eventsCheck( + 'sep-3415-unsubscribe-by-key', + '`events/unsubscribe` is eager cleanup; the server looks the subscription up by the same compound key used for idempotent upsert on subscribe.', + 'SUCCESS', + { details: { id: first.id } } + ) + ); + + const emptyDescription = 'The result is an empty `Result`.'; + if ('error' in byKey) { + out.push( + untestableCheck( + 'sep-3415-unsubscribe-empty-result', + 'sep-3415-unsubscribe-empty-result', + emptyDescription, + 'Unsubscribing by key failed, so there is no result to inspect.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + } else { + // `resultType` and `_meta` belong to every Result, so an empty one may + // still carry them; anything else is a member the SEP does not define. + const extra = + byKey.result !== null && typeof byKey.result === 'object' + ? Object.keys(byKey.result).filter( + (k) => k !== 'resultType' && k !== '_meta' + ) + : null; + out.push( + extra === null + ? eventsCheck( + 'sep-3415-unsubscribe-empty-result', + emptyDescription, + 'FAILURE', + { + errorMessage: `The unsubscribe result is ${describeValue(byKey.result)}, not an object.` + } + ) + : extra.length === 0 + ? eventsCheck( + 'sep-3415-unsubscribe-empty-result', + emptyDescription, + 'SUCCESS' + ) + : eventsCheck( + 'sep-3415-unsubscribe-empty-result', + emptyDescription, + 'FAILURE', + { + errorMessage: `The unsubscribe result carries ${extra.map((k) => `\`${k}\``).join(', ')}, which an empty \`Result\` does not define.`, + details: { result: byKey.result } + } + ) + ); + } + + // Both results this scenario holds: the subscribe that started it and the + // unsubscribe that just ended it. + const observed: ObservedResult[] = [ + { method: EVENTS_SUBSCRIBE_METHOD, result: first } + ]; + if (!('error' in byKey)) { + observed.push({ + method: EVENTS_UNSUBSCRIBE_METHOD, + result: byKey.result + }); + } + out.push(resultTypeCheck(observed)); + + // A key the server cannot hold: never subscribed, and now also the key + // that was just torn down. + const unknown = await unsubscribe({ + name, + arguments: args, + delivery: { mode: 'webhook', url: `${CALLBACK_BASE}/never-subscribed` } + }); + out.push( + 'error' in unknown + ? unknown.error.code === EVENTS_NOT_FOUND + ? eventsCheck( + 'sep-3415-unsubscribe-unknown-not-found', + 'No subscription matching the key on `events/unsubscribe` returns `-32023 NotFound`.', + 'SUCCESS', + { details: { code: unknown.error.code } } + ) + : eventsCheck( + 'sep-3415-unsubscribe-unknown-not-found', + 'No subscription matching the key on `events/unsubscribe` returns `-32023 NotFound`.', + 'WARNING', + { + errorMessage: `An unknown subscription key answered ${unknown.error.code} ${unknown.error.message}, where the document names \`-32023 NotFound\`.` + } + ) + : eventsCheck( + 'sep-3415-unsubscribe-unknown-not-found', + 'No subscription matching the key on `events/unsubscribe` returns `-32023 NotFound`.', + 'FAILURE', + { + errorMessage: + 'Unsubscribing a key that was never subscribed succeeded. A client cannot tell teardown from a no-op, and a typo in the key reads as success.' + } + ) + ); + + return out; + } +} + +/** Keep the first check emitted per id, so a fallback path cannot double-report. */ +function dedupe(checks: ConformanceCheck[]): ConformanceCheck[] { + const seen = new Set(); + return checks.filter((c) => { + if (seen.has(c.id)) return false; + seen.add(c.id); + return true; + }); +} diff --git a/src/seps/sep-3415.yaml b/src/seps/sep-3415.yaml new file mode 100644 index 00000000..c1c219de --- /dev/null +++ b/src/seps/sep-3415.yaml @@ -0,0 +1,1001 @@ +# spec_source: modelcontextprotocol/modelcontextprotocol@c47fd2451c91ad11708dd498ddee060a1d73aedc seps/0000-events-extension.md +# extracted: 2026-09-15 against the design sketch; every row re-quoted against SEP-3415 at c47fd245 on 2026-10-14 +# +# ############################################################################ +# # MCP Events is SEP-3415 (modelcontextprotocol/modelcontextprotocol#3415). # +# # Every row quotes the SEP at c47fd245 and carries a `url` to its section. # +# ############################################################################ +# +# SEP numbers are PR numbers in modelcontextprotocol/modelcontextprotocol. +# Peter Alexander opened PR 3415 on 2026-10-05, writing the design sketch up +# in SEP format, so this file and every check ID took that number in place of +# the 9999 placeholder they carried while no PR existed. The file keeps its +# `0000-events-extension.md` name upstream until the SEP is accepted, but the +# number does not change. +# +# The rows were first written against the design sketch, and the SEP restates +# the sketch rather than quoting it: only 11 of 165 row texts survived +# verbatim, even where the rule was unchanged. So every row was re-quoted +# (panyam/mcpkit#1502). A row's `text` is an exact span of the SEP, and its +# `url` is the section that span sits under, both checked mechanically +# against the pinned file. Three conventions came out of that: +# +# - A row backed by a table quotes the table row as its cells, the last one +# after a colon (e.g. "`NotFound` `-32023`: A referenced entity ..."). The +# SEP gives the error-code numbers only in its Error Handling table, so the +# code rows all read this way. +# - Two rules the SEP states only inside its JSON examples +# (sep-3415-list-pagination, sep-3415-descriptor-meta) quote the example's +# comment, and two (sep-3415-list-implemented, sep-3415-descriptor-name) +# quote the Abstract, which is the only place the SEP says them in prose. +# - Seven excluded rows quote Appendix A, the SEP's SDK guidance, which is +# non-normative. They stay listed for traceability and could not back a +# graded row. +# +# What the re-extraction changed besides wording: +# +# - sep-3415-descriptor-description is gone. The SEP never states that a +# descriptor carries a `description`; the field appears only in the JSON +# example and in the list of fields whose change triggers `list_changed`. +# - Two rows are new: sep-3415-subscribe-refresh-before-present +# ("`refreshBefore` (response, always present, nullable)") and +# sep-3415-unsubscribe-empty-result ("The result is an empty `Result`"). +# - Five graded rows now quote a rule the SEP states differently from the +# sketch, and their checks have not caught up (panyam/mcpkit#1502): +# sep-3415-capability-client-declaration (declared per request in `_meta` +# on the 2026-07-28 wire), sep-3415-capability-list-changed-gated and +# sep-3415-list-changed-notification (delivered on `subscriptions/listen` +# to clients that opted in), sep-3415-delivery-410-non-retryable (the +# subscription itself MUST survive a 410) and +# sep-3415-delivery-status-last-error-category (the MUST NOT now covers raw +# headers and status lines too). The first emits no check today. The two +# list_changed rows are graded over `subscriptions/listen` in +# events-discovery since panyam/mcpkit#1505. +# +# The six capability rows quote SEP-3415's Capability Negotiation section, +# which put the declaration in Extension Negotiation. The sketch's `main` +# still says top-level `capabilities.events`; its PR 7, which would have +# moved it, closed unmerged on 2026-09-28. +# +# provenance: first extracted against the merged design sketch on `main`, which +# landed 2026-09-08. Two spec commits land inside that extraction: +# `197c32b4` (2026-05-10) renamed the duration fields to `nextPollMs` and +# `maxAgeMs`, and `28ec35e9` (2026-09-04) added event-type removal and +# breaking-change termination. The SEP carries both. +# +# The SEP is a single markdown file under `seps/`, not a +# `docs/specification/draft/*.mdx` diff, so `new-sep`'s `--spec-path` +# (src/new-sep/index.ts:19 hard-requires that prefix) still cannot point at +# it. `spec_url` and the per-row urls are pinned to the commit instead, so +# they keep resolving when the PR moves. Re-extract when the SEP's head +# moves past c47fd245. +# +# coverage: 136 declared checks plus 32 excluded rows. On 2026-10-15 +# sep-3415-stream-mode-unsupported was added (#1417), and +# sep-3415-schema-evolution-additive and sep-3415-breaking-change-new-name +# moved to excluded (#1465): both are SHOULDs about how a server chooses to +# evolve its catalog, which a harness forcing a change cannot observe. The +# 2026-10-14 re-extraction before that dropped +# sep-3415-descriptor-description and added +# sep-3415-subscribe-refresh-before-present and +# sep-3415-unsubscribe-empty-result. +# +# A keyword sweep of SEP-3415 at c47fd245, outside code blocks and before the +# non-normative Appendix A, finds 135 RFC 2119 keyword occurrences: 33 MUST, +# 10 MUST NOT, 3 REQUIRED, 46 SHOULD, 9 SHOULD NOT (101 normative), plus 33 +# MAY and 1 OPTIONAL. The design sketch these rows were first extracted from +# had 144 (109 normative); the SEP states fewer rules with keywords and more +# as tables and examples. +# +# Only 46 of the declared rows quote a sentence carrying a keyword. The other +# 91 are shape requirements the document states declaratively — the +# `EventOccurrence` field table, the error-code table, the `events/list` +# descriptor fields, the response payloads. They are normative and directly +# testable, they are simply not written with a keyword, which is why the row +# count exceeds the sentence count in the sweep. One keyword sentence also +# frequently carries several distinct obligations (the Standard Webhooks +# signature sentence, the subscription-key sentence), which pushes the same +# way. +# +# severity rule, since the usual "follow the keyword" mapping is undefined +# for a shape row: MUST / MUST NOT / REQUIRED -> FAILURE, SHOULD / +# SHOULD NOT -> WARNING. For a shape row, a field the document +# marks Required in a schema table, or a wire fact a client cannot work +# without (an error code, a method existing at all), is FAILURE; everything +# else drawn from an example payload is WARNING. +# +# Pure MAY and OPTIONAL sentences get no row at all, per the handoff +# disposition table. The five that initially got one are demoted to +# `excluded:` at the foot of this file rather than deleted, so the sweep +# stays auditable against the document. +# +# excluded rather than declared: client-SDK and host obligations (poll floor, +# payload sanitization, policy evaluation), receiver obligations the harness +# implements rather than grades, server-SDK implementation guidance from the +# two "SDK Guidance" sections, and the WAF/deployment profile advisories. +# None is a server wire obligation, so declaring them would inflate the +# denominator with rows no server-side scenario could ever emit. +# +# deliberately not declared here: pagination semantics for `events/list`. The +# sketch defers them ("same semantics as tools/list etc."), so the obligation +# is the base protocol's and belongs to the core pagination scenarios. +# `sep-3415-list-pagination` declares only the part this document adds, which +# is that `events/list` participates in that scheme at all. +# +# An `events/poll` against an unknown name is stated twice, once under Error +# Codes and once as the poll consequence of an event type having been +# removed. Only `sep-3415-removal-poll-not-found` is declared for it; a +# second row would double-count one probe. +# +# `## Key Design Decisions`, `## Open Questions` and `## What Is NOT in v1` +# are rationale rather than requirements and produce no rows. The design +# table restates the webhook secret, TTL and TLS rules already declared above +# it. +# +# backing_scenarios: server ClientScenarios under src/scenarios/server/events/ +# emit the check IDs below (a row is "tested" once a scenario emits its ID; +# see src/traceability/). All five now exist and together emit 126 of the 137 +# rows. sep-3415-result-type-complete is emitted by four of them (discovery, +# poll, push and webhook), each grading the results it already holds, so it +# counts once here and once in each scenario below: +# discovery.ts (events-discovery), 17 rows — the three capability rows, the +# two sep-3415-list-* rows, the six sep-3415-descriptor-* rows, +# sep-3415-error-not-found plus sep-3415-error-server-range (both graded +# off one unknown-name probe), and sep-3415-fallback-method-not-found, +# which only a server that does not offer the extension can answer, plus +# sep-3415-result-type-complete on the `events/list` result, plus the +# two list_changed rows: sep-3415-capability-list-changed-gated off the +# `subscriptions/listen` acknowledgement, and +# sep-3415-list-changed-notification off a catalog change made through the +# fixture's events_conformance_catalog control. +# poll.ts (events-poll), 35 rows — the sep-3415-poll-* rows, the +# sep-3415-occurrence-* rows, the sep-3415-cursor-* / +# sep-3415-max-age-* / sep-3415-truncated-* rows reachable through poll, +# sep-3415-removal-poll-not-found (the poll leg of the removal rules), and +# sep-3415-error-invalid-params, which rides the same probe as +# sep-3415-poll-invalid-arguments, plus sep-3415-result-type-complete on +# the bootstrap poll. +# push.ts (events-push), 19 rows — the sep-3415-stream-* rows plus +# sep-3415-error-resource-exhausted, graded when a server refuses one of +# the three concurrent streams for a quota. It holds a +# real SSE stream open through stream.ts, because conn.request() resolves +# on the response for its id and a push stream withholds that until the +# subscription ends. sep-3415-result-type-complete is graded on the +# stream's final frame, and is a WARNING untestable when the server never +# closes the stream itself, since clients MUST NOT depend on that frame. +# webhook.ts (events-webhook), 28 rows — sep-3415-subscribe-*, +# sep-3415-ttl-*, sep-3415-unsubscribe-*, sep-3415-error-unsupported, and +# sep-3415-result-type-complete on the subscribe and unsubscribe results. +# webhook-delivery.ts (events-webhook-delivery), 29 rows — +# sep-3415-delivery-*, sep-3415-verification-*, sep-3415-ssrf-*, +# sep-3415-envelope-* and sep-3415-error-callback-endpoint-error, graded +# from an HTTP receiver the harness runs. The receiver publishes +# /.well-known/mcp-webhook-receiver.json over one path prefix and no other, +# so a server that takes the document path and one that handshakes are both +# exercised in a single run; EVENTS_RECEIVER_WELL_KNOWN=0 withholds it. +# +# negative controls: src/scenarios/server/events/negative*.test.ts, one file per +# scenario over a shared fixture in negative-fixture.ts. A green run against +# kitchen-sink proves a check does not false-positive; only a fixture broken in +# exactly one way proves it catches anything. Every divergence listed below has +# a case there, so a red run against a real server can be trusted rather than +# re-derived. +# +# The fixture can also do the four things no client can ask a real server for +# (fail upstream, lose its replay window, terminate a subscription, close a +# stream itself) plus the gap and terminated control envelopes. Those rows +# report untestable against both implementations, so the controls are the only +# evidence those checks work at all. +# +# The 11 rows nothing emits yet, and what each needs: +# sep-3415-schema-evolution-additive, sep-3415-breaking-change-new-name +# and the four sep-3415-removal-* rows need a catalog that changes mid-run. +# events_conformance_catalog adds and removes types now (it drives +# sep-3415-list-changed-notification), which is enough for the removal +# rows once a scenario subscribes before the remove. The two +# schema-evolution rows also need it to alter a type in place +# (panyam/mcpkit#1465). +# sep-3415-error-forbidden is the last error-code table row with no probe: +# it needs a principal the harness can be refused as. The other three are +# claimed off the rule that provokes each, with details.gradedBy naming it: +# -32602 off the poll invalid-arguments probe, -32013 off the concurrent +# streams, -32015 off the wrong-challenge callback. +# sep-3415-capability-client-declaration is a MAY about what a client +# declares, so no server-side scenario can emit it. Declared for now; the +# honest place for it is `excluded:` beside the other MAY rows. +# sep-3415-authz-subscribe-time and -delivery-time-reverify need two +# principals with different permissions. +# sep-3415-payload-minimality is a SHOULD about payload content, which +# sep-3415-delivery-body-size measures the observable half of. +# +# running them: events-push needs `--timeout 60000`, because it watches an idle +# stream for 35s and the document allows a 30s heartbeat cadence. A shorter +# window cannot tell a silent server from a slow one, so under 30s the +# heartbeat rows report untestable instead of failing. EVENTS_PUSH_WATCH_MS +# overrides it. +# +# events-webhook-delivery needs a callback the server under test can reach: +# set EVENTS_WEBHOOK_CALLBACK_BASE to a public https base URL forwarding to +# the harness. Without one it points the server at a loopback receiver, which +# is the SSRF probe rather than a degraded run — a conformant server refuses +# it, and one that delivers there fails the SSRF rows and supplies real +# deliveries for everything else. +# +# measured against mcpkit, 2026-09-17, examples/events/kitchen-sink at +# `d2950655`: events-discovery 8/11, events-poll 26/29. The poll number was +# 18/28 on 2026-09-15; the seven `sep-3415-occurrence-*` rows now pass +# because the scenario polls forward from the bootstrap cursor rather than +# grading them off the bootstrap response, and `sep-3415-poll-invalid- +# arguments` moved from a failure to untestable. Nothing in mcpkit changed +# between the two runs. Each divergence below is a real gap rather than +# something the suite should accommodate. Do not soften these rows to make +# mcpkit pass; catching them is the point. +# +# sep-3415-capability-events-object — the server answers `events/list` but +# declares nothing, so a client that reads capabilities before calling never +# finds the surface. Not previously tracked. This is also why the scenario +# asks before it skips: a plain "undeclared optional capability" SKIP +# reports this server as a clean run. Closed by mcpkit 1416 (declaring +# top-level, which the document then said) and corrected to the extensions +# map by mcpkit 1421 once upstream PR 7 moved it. +# sep-3415-poll-next-poll-ms — `nextPollSeconds` persists (events.go:504) +# after `197c32b4` renamed it, and `maxAge` is still in seconds +# (events.go:544). mcpkit's own wire_shape_test.go:44 asserts the +# pre-rename name. Gap G31. +# sep-3415-descriptor-delivery-subset — `events.topology` advertises an +# empty `delivery`, where the document requires a non-empty subset of +# poll/push/webhook. That source is the substitute for the missing +# `notifications/events/list_changed`. Gap G34. +# sep-3415-descriptor-input-schema — `events.topology` carries no +# `inputSchema`. mcpkit 1381 added one to the three real sources, so this +# and the delivery row above now have the same single cause. The poll +# scenario picks `alert.fired`, which has an `inputSchema` with no typed +# property, so `sep-3415-poll-invalid-arguments` reports untestable: there +# is no constraint the harness can violate on purpose. +# sep-3415-poll-events-array — `events` is omitted rather than returned as +# an empty array when nothing happened. Not previously tracked. +# sep-3415-poll-mode-unsupported — `events/poll` answers for an event type +# whose `delivery` does not list `poll`, where the document requires +# `-32014 Unsupported`. Not previously tracked. +# +# The removal and termination rules (gap G30) are declared here but not yet +# exercised: `RemoveSource` (registry.go:182) fires topology only and +# `UnsupportedData` (errors.go:90) has no `reason` field, but driving that +# path needs a server that can drop an event type mid-run, which waits on +# the push scenario. +# +# measured against a second implementation, 2026-09-17: Peter Alexander's +# `https://metronome-mcp.fly.dev/mcp`, announced 2026-09-10 and written by +# the author of the design sketch. events-discovery 10/11, events-poll 27/29. +# It serves one event type, `metronome.tick`, which offers all three delivery +# modes. +# +# Running it is not a one-liner. Three things differ from a local fixture: +# +# Events are only on the 2026-07-28 stateless wire. Over 2025-11-25 the +# server answers `events/list` with -32601. The runner picks the wire from +# the spec version, and the default already lands on stateless, so no flag +# is needed — but a 2025-11-25 run reports the whole extension as absent. +# It is behind OAuth, and `server` has no way to send a token. Its +# authorization server is a demo that verifies no identity: register a +# client at /register, approve the consent form, exchange the code with +# PKCE. Then put a local proxy in front that adds the header and point +# --url at the proxy. An auth option on the runner would remove all of +# this, and the webhook scenarios will need one anyway, since a webhook +# subscription requires an authenticated principal. +# Its one event type has required `inputSchema` properties. See the +# `minimalArguments` note below. +# +# Two divergences, neither previously known. Both are now resolved, and the +# way they resolved is the argument for running a suite against more than one +# implementation: +# +# sep-3415-capability-events-object — metronome declared events under +# `capabilities.extensions["io.modelcontextprotocol/events"]`, the SEP-2133 +# extensions map, where this document put it top-level under +# `capabilities`. Raised with the author 2026-09-22 rather than resolved by +# picking a side. **The document was wrong**: extension negotiation is where +# it belongs, and upstream PR 7 moves the sketch there. metronome was right +# all along and mcpkit's conformance was an artifact of following a document +# that did not yet say what its author meant. The rows above track PR 7; the +# banner at the head of this file names them and says what to do on merge. +# sep-3415-poll-invalid-arguments — arguments that violate `inputSchema` +# answered -32603 Internal error, where the document requires -32602 +# InvalidParams. Read as a framework default surfacing rather than a +# decision, which is what it was. Fixed by the author 2026-09-22. +# +# `sep-3415-poll-mode-unsupported` reports untestable here, for the opposite +# reason it fails against mcpkit: every event type metronome serves offers +# poll, so there is no type to probe the unsupported-mode path with. The two +# implementations cover each other's blind spot, which is the argument for +# keeping both targets rather than picking one. +# +# two suite bugs the second implementation found, both fixed here: +# +# `minimalArguments` (helpers.ts) returned undefined for any `inputSchema` +# with a required property, on the reasoning that every schema in the +# document is filters and transforms with nothing required. That holds for +# the examples in the document, not for a real server: `metronome.tick` +# requires `periodSeconds` and `label`, and all 33 poll rows reported +# untestable, scoring 1/34. It now derives a value per required property +# from the schema — `default`, `const`, first `enum` or `examples` entry, +# then `minimum` for numbers or a fixed string — and still gives up, rather +# than guessing, when a required property offers nothing to go on. +# The `sep-3415-occurrence-*` rows were graded off the bootstrap poll, which +# passes `cursor: null`. `cursor: null` means start from now, so a +# conformant server returns no events and the seven shape rows could never +# pass: the suite required the server to violate +# `sep-3415-cursor-null-starts-from-now` to score them. They now poll +# forward from the returned cursor, waiting `nextPollMs` between attempts +# (clamped to 250ms-2s) up to a 6s ceiling, and still report untestable +# against a genuinely quiet server. This is why the mcpkit poll number +# moved without mcpkit changing. +# +# Both bugs were invisible against kitchen-sink alone. The first needs a +# server whose schema requires something; the second needs one that does not +# replay on a null cursor. This is the same pattern as the SEP-2640 suite, +# where the two bugs that mattered came from running against something that +# was not ours. +# measured across all five scenarios, 2026-09-20: +# +# kitchen-sink (d2950655) metronome-mcp.fly.dev +# events-discovery 8/11 10/11 +# events-poll 26/29 27/29 +# events-push 11/15 13/15 +# events-webhook 17/23 20/23 +# events-webhook-delivery 12/21 3/27 +# +# metronome's webhook-delivery number is not a result about metronome. It +# refuses the loopback callback, correctly, so 24 rows report untestable for +# want of a routable one. Point EVENTS_WEBHOOK_CALLBACK_BASE at a tunnel and +# that run becomes meaningful. +# +# new divergences from the three later scenarios, none previously tracked, +# all in mcpkit rather than in the document: +# +# sep-3415-stream-subscription-id-meta — kitchen-sink puts the correlation id +# in `params.requestId`, where the document requires +# `params._meta["io.modelcontextprotocol/subscriptionId"]`. A client holding +# two streams cannot route by what the document tells it to read. metronome +# carries it correctly, which is what identified this as a defect rather +# than an ambiguity in the sketch. +# sep-3415-subscribe-url-https-required / -url-non-https-rejected — +# kitchen-sink accepts an `http://` callback URL and subscribes to it. +# sep-3415-unsubscribe-unknown-not-found — kitchen-sink answers success for a +# subscription key it never held, so a client cannot tell teardown from a +# typo. +# sep-3415-delivery-signature-formula — kitchen-sink computes the HMAC with +# the literal `whsec_...` string as the key, where the document says the key +# is the base64-decoded bytes after the prefix. No Standard Webhooks +# receiver verifies these signatures. Checked by hand against four candidate +# formulas before this was written down. +# sep-3415-ssrf-validate-callback-url / -reject-non-routable — the fixture +# accepts and then delivers to `http://127.0.0.1`. This one is fixture +# configuration rather than a library defect: kitchen-sink passes +# `events.WithWebhookAllowPrivateNetworks(true)` (main.go:151) so `make +# demo` can reach a local receiver, and that single flag disables both the +# subscribe-time hostname check and the dial-time IP check. mcpkit's +# default path rejects loopback, RFC1918, link-local, ULA and the rest +# (webhook.go isBlockedIP). The discord, telegram and whole-enchilada demos +# set the same flag. +# +# The row still fails, and should: the suite grades the server it was +# pointed at, and the server it was pointed at delivered to loopback. But +# read it as "the conformance target runs with SSRF protection off", not as +# "mcpkit has no SSRF protection". Pointing testconf-events at a fixture +# built without that flag is the fix, and would also make the row mean +# something. +# sep-3415-verification-required-before-delivery — kitchen-sink delivers +# without the challenge handshake, because the handshake does not exist: +# control.go carries only the `gap` and `terminated` discriminators, and +# the `challenge_failed` error bucket is declared and never assigned. +# Tracked as mcpkit issue 490, with DEPLOYMENT.md calling the current state +# verification-stub-only. +# +# the signature and verification findings are the same shape of problem and +# worth reading together: nothing yet proves a callback endpoint wanted the +# deliveries, and what arrives there cannot be authenticated by its signature. +# metronome fails neither. +# +# root causes, read out of mcpkit at origin/main (5cfb7c51) rather than +# guessed, because "why" decides whether a row is worth arguing about: +# +# The correlation-id, https and unsubscribe rows are all library-side in +# experimental/ext/events. The correlation id predates the convention it +# should follow: stream.go names the field `requestId`, mirroring the +# sketch's own push examples, while mcpkit implements the SEP-2575 `_meta` +# spelling elsewhere (core/stateless.go MetaKeySubscriptionID). The id it +# would carry already exists server-side and is simply never put on the +# wire. `ValidateWebhookURL` allows `http` deliberately for demo +# ergonomics, documented in DEPLOYMENT.md, and maps failures to -32015 +# rather than -32602 besides. The unsubscribe row is an unfinished feature +# rather than a decision: the registry already computes whether the key was +# found and the handler discards it, and wire_shape_test.go describes the +# NotFound answer as future work. +# The signature row is a chronology accident and the widest to fix. Signing +# landed 2026-04-29, when the secret was an opaque server-minted token; the +# `whsec_` + base64 format arrived two days later, and key derivation was +# never revisited. The same literal-string key appears in both verifiers, +# the Go and Python clients, the whole-enchilada receiver and the telegram +# tests, so server and clients are mutually consistent and all five call +# sites are non-conformant together. That is exactly the failure mode a +# conformance suite exists to catch: the implementation's own tests cannot +# see it. +# +# None of these six is in mcpkit issue 1380, which tracks a disjoint set from +# the earlier run. Only the verification gap has an issue (490). The package +# carries an EXPERIMENTAL banner and sits outside the per-PR CI matrix, which +# is a large part of why they survived. +# +# two harness bugs the second implementation found, both fixed: +# +# The `sep-3415-occurrence-*` rows were graded off the bootstrap poll, which +# passes `cursor: null` and therefore returns nothing on a conformant +# server. See the 2026-09-17 note below. +# `sep-3415-delivery-retry-regenerates-signature` folded signature validity +# into its freshness check, so a server with the wrong signature formula was +# reported as reusing timestamps it had not reused. Freshness is now graded +# on distinct timestamps and signatures alone, and validity belongs to +# sep-3415-delivery-signature-formula. +# +sep: 3415 +# SEP-3415 pinned at the commit every row quotes. See the header. +spec_url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md +requirements: + # === Capability Declaration === + - check: sep-3415-capability-events-object + text: "the identifier appears as a key in the `extensions` field of capabilities, mapped to the extension's settings object." + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#capability-negotiation + - check: sep-3415-capability-list-changed-flag + text: "The server's settings object has one member: `listChanged` (boolean, optional, default `false`), which says whether the server sends `notifications/events/list_changed`" + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#capability-negotiation + - check: sep-3415-capability-empty-settings + text: 'An empty settings object declares event support with no list-change notifications.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#capability-negotiation + - check: sep-3415-capability-client-declaration + text: 'A client MAY declare the same identifier with an empty settings object, in its per-request capabilities, to indicate that it understands the extension:' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#capability-negotiation + - check: sep-3415-capability-list-changed-gated + text: 'A server that did not declare `listChanged: true` omits it from the acknowledged filter.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#dynamic-event-types + - check: sep-3415-fallback-method-not-found + text: 'a server that does not offer it answers any `events/*` request with `-32601` (Method not found).' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#capability-negotiation + + # === Listing Available Events === + - check: sep-3415-list-implemented + text: 'A server lists its event types with `events/list`.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#abstract + - check: sep-3415-list-pagination + text: '"nextCursor": "...", // present when more pages are available; same semantics as tools/list' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#response + - check: sep-3415-descriptor-name + text: 'Each event type has a name, an `inputSchema` for subscription arguments, a `payloadSchema`, and the delivery modes it supports.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#abstract + - check: sep-3415-descriptor-delivery-subset + text: '`delivery` lists the delivery modes the event type supports. It is any non-empty subset of `"poll"`, `"push"` and `"webhook"`. No mode is mandatory.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#response + - check: sep-3415-descriptor-input-schema + text: '`inputSchema` is a JSON Schema for valid subscription arguments.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#response + - check: sep-3415-descriptor-payload-schema + text: '`payloadSchema` is a JSON Schema for `data` in delivered events.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#response + - check: sep-3415-descriptor-meta + text: '"_meta": { "...": "..." }, // optional; same semantics as on Tool/Resource/Prompt' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#response + + # === Dynamic Event Types === + - check: sep-3415-list-changed-notification + text: 'If the set of available event types changes at runtime, or the descriptor of any of them changes (`description`, `delivery`, `inputSchema`, `payloadSchema`), a server that declared `listChanged: true` sends `notifications/events/list_changed`.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#dynamic-event-types + + # === Event Type Removal and Breaking Changes (spec commit 28ec35e9) === + - check: sep-3415-removal-terminates-subscriptions + text: "The server SHOULD end them with each mode's termination signal (see [Subscription Termination](#subscription-termination)): `notifications/events/terminated` on push streams, and a `terminated` envelope to webhook subscriptions." + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#event-type-removal-and-breaking-changes + - check: sep-3415-removal-error-not-found + text: '`NotFound` with `data: {"kind": "event"}` when the type was removed.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#event-type-removal-and-breaking-changes + - check: sep-3415-removal-error-schema-changed + text: '`Unsupported` with `data: {"feature": "payloadSchema" | "inputSchema", "reason": "schema_changed"}` when it was changed in place. It is not `Forbidden`, because the principal''s access is unchanged.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#event-type-removal-and-breaking-changes + - check: sep-3415-removal-additive-no-terminate + text: 'Purely additive changes MUST NOT terminate subscriptions.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#event-type-removal-and-breaking-changes + - check: sep-3415-removal-poll-not-found + text: 'Poll holds no server-side subscription to terminate. A poll against a removed name already returns `NotFound`.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#event-type-removal-and-breaking-changes + + # === Error Codes === + - check: sep-3415-error-invalid-params + text: "`InvalidParams` `-32602`: The request is statically invalid: the arguments do not match the event's `inputSchema`, the callback `delivery.url` is malformed or not `https`, or `delivery.secret` is not a valid `whsec_` value. This is the standard JSON-RPC code." + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#error-handling + - check: sep-3415-error-not-found + text: '`NotFound` `-32023`: A referenced entity does not exist: an unknown event name, or no subscription that matches the key on `events/unsubscribe`. The method implies which. `data.kind` (`"event"` | `"subscription"`) MAY disambiguate.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#error-handling + - check: sep-3415-error-forbidden + text: '`Forbidden` `-32024`: The authenticated principal is not permitted for this combination of event and arguments, or its access was revoked.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#error-handling + - check: sep-3415-error-resource-exhausted + text: '`ResourceExhausted` `-32025`: A server-imposed limit or quota was reached. `data.limit` names it (for example `"subscriptions"`), and `data.max` MAY give the ceiling.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#error-handling + - check: sep-3415-error-unsupported + text: '`Unsupported` `-32026`: The request is well-formed, but a requested capability or option is not supported here, for example a delivery mode that the event type does not offer. `data` identifies it (for example `{ "feature": "deliveryMode", "value": "push" }`).' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#error-handling + - check: sep-3415-error-callback-endpoint-error + text: '`CallbackEndpointError` `-32027`: A client-supplied callback endpoint failed verification or could not be reached (webhook mode only). `data.reason` is one of the `lastError` categories (`connection_refused`, `timeout`, `tls_error`, `http_4xx`, `http_5xx`, `challenge_failed`).' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#error-handling + - check: sep-3415-error-server-range + text: 'The core specification partitions the JSON-RPC implementation-defined range: `-32000` to `-32019` is left to implementations and is never defined by the specification, and `-32020` to `-32099` is allocated sequentially by the specification. This SEP requests the next five codes in the allocated range.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#error-handling + + # === Results (SEP-3415 conventions) === + - check: sep-3415-result-type-complete + text: 'Every result this extension defines is a standard `Result` and carries `resultType: "complete"`.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#specification + + # === Poll-Based Delivery === + - check: sep-3415-poll-implemented + text: 'The client calls `events/poll` with the event name, arguments and cursor.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#delivery-modes + - check: sep-3415-poll-one-subscription-per-request + text: 'Each `events/poll` request carries one subscription.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#request-eventspoll + - check: sep-3415-poll-bootstraps-subscription + text: 'There is no separate subscribe step: the first poll with a `null` cursor bootstraps the subscription.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#delivery-modes + - check: sep-3415-poll-events-array + text: 'An empty `events` array means nothing happened.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#response-1 + - check: sep-3415-poll-response-cursor + text: 'Poll covers this inherently, because every response carries `cursor`, including when `events` is empty.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#cursor-lifecycle + - check: sep-3415-poll-next-poll-ms + text: '`nextPollMs` lets the server adjust polling frequency, for example to back off when it is rate-limited upstream or to speed up when it sees activity.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#response-1 + - check: sep-3415-poll-next-poll-ms-ignored-when-has-more + text: 'It is ignored when `hasMore` is `true`.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#response-1 + - check: sep-3415-poll-has-more + text: '`hasMore` says whether more events are available beyond the returned batch.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#response-1 + - check: sep-3415-poll-max-events-cap + text: '`maxEvents` is an optional cap on the number of events returned. If more events are available than the cap, the server returns a partial batch with an intermediate cursor and sets `hasMore: true`.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#response-1 + - check: sep-3415-poll-stateless-request + text: 'Each poll request is self-contained: the client provides the event name, arguments and cursor, and the server does not need to remember earlier polls to answer it.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#response-1 + - check: sep-3415-poll-errors-are-jsonrpc + text: 'Errors (`NotFound`, `Forbidden`, `InvalidParams`, `Unsupported`) are returned as a standard JSON-RPC error response.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#response-1 + - check: sep-3415-poll-invalid-arguments + text: "`InvalidParams` `-32602`: The request is statically invalid: the arguments do not match the event's `inputSchema`, the callback `delivery.url` is malformed or not `https`, or `delivery.secret` is not a valid `whsec_` value. This is the standard JSON-RPC code." + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#error-handling + - check: sep-3415-poll-mode-unsupported + text: '`Unsupported` `-32026`: The request is well-formed, but a requested capability or option is not supported here, for example a delivery mode that the event type does not offer. `data` identifies it (for example `{ "feature": "deliveryMode", "value": "push" }`).' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#error-handling + + # === EventOccurrence schema === + - check: sep-3415-occurrence-event-id + text: '`eventId` string yes: Stable identifier for deduplication.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#event-occurrences + - check: sep-3415-occurrence-name + text: '`name` string yes: Event type name.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#event-occurrences + - check: sep-3415-occurrence-timestamp + text: '`timestamp` string (ISO 8601) yes: When the event occurred.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#event-occurrences + - check: sep-3415-occurrence-data + text: "`data` object yes: Payload that conforms to the event type's `payloadSchema`." + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#event-occurrences + - check: sep-3415-occurrence-cursor-optional + text: '`cursor` string | null no: Subscription position after this event. Push and webhook only; poll carries the cursor at the response level. `null` when the event type does not support replay (see [Cursor Lifecycle](#cursor-lifecycle)).' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#event-occurrences + - check: sep-3415-occurrence-meta-ungoverned + text: '`_meta` object no: Reserved for protocol and extension metadata, consistent with `_meta` on other MCP types. Not governed by `payloadSchema`.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#event-occurrences + - check: sep-3415-occurrence-event-id-from-upstream + text: 'When the upstream source provides a stable event identifier (a Stripe `evt_*`, a GitHub delivery GUID, a Kafka offset, a Gmail message ID), the server SHOULD use that value as `eventId`.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#event-occurrences + + # === Cursor Lifecycle === + - check: sep-3415-cursor-opaque + text: 'Cursors are opaque strings managed by the server. They represent a position in the event stream.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#cursor-lifecycle + - check: sep-3415-cursor-null-starts-from-now + text: 'Passing `cursor: null` in any mode means "start from now". The server returns a fresh cursor for the current position, or `null` if the event type does not support replay. No historical events are replayed.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#cursor-lifecycle + - check: sep-3415-cursor-absent-equals-null + text: 'An absent `cursor` field MUST be treated identically to an explicit `cursor: null`. A sender MAY omit the field in place of writing `null`, and a receiver MUST NOT fail because it is missing.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#cursor-lifecycle + - check: sep-3415-cursor-consistency + text: 'Servers SHOULD be consistent: an event type that ever returns a non-null cursor SHOULD always do so, and one that returns `null` SHOULD always return `null`.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#cursor-lifecycle + - check: sep-3415-cursor-advances-when-quiet + text: "The client's persisted cursor has to advance during quiet periods, so that it does not fall outside the upstream's retention window when nothing is happening. Poll covers this inherently, because every response carries `cursor`, including when `events` is empty." + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#cursor-lifecycle + - check: sep-3415-max-age-ms-floor + text: 'All three modes accept an optional `maxAgeMs` (integer milliseconds) alongside `cursor`. When it is present, the server begins replay from whichever is _later_: the supplied `cursor`, or `now − maxAgeMs`.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#cursor-lifecycle + - check: sep-3415-max-age-ms-sets-truncated + text: 'If the floor advances past the cursor, the server SHOULD set `truncated: true` on the first response (the poll result, `notifications/events/active`, or the webhook subscribe response), so that the client knows older events were skipped.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#cursor-lifecycle + - check: sep-3415-max-age-ms-ignored-when-cursor-null + text: '`maxAgeMs` is ignored when `cursor` is `null`, because `null` already means "now" and replay from before the subscription''s start is out of scope.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#cursor-lifecycle + - check: sep-3415-replay-ceiling-sets-truncated + text: 'Servers MAY also apply their own replay ceiling independent of `maxAgeMs`, and MUST signal it with `truncated: true` when they do.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#cursor-lifecycle + - check: sep-3415-truncated-returns-fresh-cursor + text: 'In every case the server resets to a position it can serve from, returns that position as the fresh `cursor` alongside `truncated: true`, and continues. The client never reconnects or resubscribes in response.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#cursor-lifecycle + - check: sep-3415-truncated-poll-never-an-error + text: 'Poll: The result body: `{events:[], cursor:, truncated:true, hasMore, nextPollMs}`. Never a JSON-RPC error.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#cursor-lifecycle + - check: sep-3415-truncated-false-when-no-replay + text: 'For event types that do not support replay, where `cursor` is always `null`, `truncated` SHOULD be `false` and clients SHOULD ignore it.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#cursor-lifecycle + + # === Push-Based Delivery === + - check: sep-3415-stream-implemented + text: 'Push delivery uses a long-lived `events/stream` request, one per subscription. The client opens a stream for each subscription, and the server delivers events as they occur. The request is a standard JSON-RPC request with an `id`.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#push-based-delivery + - check: sep-3415-stream-error-before-open + text: 'If the subscription is invalid (`NotFound`, `Forbidden`, `InvalidParams`, `Unsupported`), the server responds immediately with a JSON-RPC error and no stream is opened.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#event-delivery + - check: sep-3415-stream-mode-unsupported + text: '`Unsupported` `-32026`: The request is well-formed, but a requested capability or option is not supported here, for example a delivery mode that the event type does not offer. `data` identifies it (for example `{ "feature": "deliveryMode", "value": "push" }`).' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#error-handling + - check: sep-3415-stream-active-confirmation + text: 'Otherwise the server confirms the subscription with `notifications/events/active`, and then delivers events as notifications.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#event-delivery + - check: sep-3415-stream-subscription-id-meta + text: 'Every `notifications/events/*` message carries the JSON-RPC `id` of the parent `events/stream` request in `params._meta["io.modelcontextprotocol/subscriptionId"]`, the correlation key the core protocol defines for subscription streams.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#event-delivery + - check: sep-3415-stream-event-notification + text: "Each entry in a poll result's `events[]`, the params of `notifications/events/event`, and the body of a webhook event delivery is an `EventOccurrence`." + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#event-occurrences + - check: sep-3415-stream-error-is-recoverable + text: '`notifications/events/error` reports a recoverable failure, such as a single failed upstream fetch. The subscription stays active, and the server retries and resumes.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#event-delivery + - check: sep-3415-stream-terminated-ends-subscription + text: 'Only `notifications/events/terminated` ends the subscription' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#event-delivery + - check: sep-3415-stream-gap-resends-active + text: "A gap, such as a cursor that fell outside the upstream's retention window, is not an error. The server sends a fresh `notifications/events/active` with `truncated: true` and a new cursor, and continues delivering" + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#event-delivery + - check: sep-3415-stream-heartbeat-required + text: 'The server MUST send periodic keepalive messages on the push stream, so that the client can tell "nothing to send" from "the connection is dead".' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#lifecycle + - check: sep-3415-stream-heartbeat-carries-cursor + text: "`cursor` is the position the server has checked up to, so the client's persisted cursor advances even when no events match (see [Cursor Lifecycle](#cursor-lifecycle)). It is `null` for event types that do not support replay." + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#lifecycle + - check: sep-3415-stream-heartbeat-interval + text: 'The server SHOULD send a heartbeat at least every 30 seconds.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#lifecycle + - check: sep-3415-stream-heartbeat-not-sse-comment + text: 'On Streamable HTTP the heartbeat is an SSE `data:` frame. The SSE comment form (`: keepalive`) is not used, because it cannot carry cursor state.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#lifecycle + - check: sep-3415-stream-final-result-shape + text: 'The `StreamEventsResult` is an empty result that carries only `resultType` and the subscription ID in `_meta`, mirroring `SubscriptionsListenResult`. It carries no information.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#lifecycle + - check: sep-3415-stream-final-result-timing + text: "It is sent whenever the server can write a final frame: on Streamable HTTP only when the server initiates the close, and on stdio at the server's discretion. Clients MUST NOT depend on receiving it." + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#lifecycle + - check: sep-3415-stream-cancel-stops-delivery + text: 'In both cases the server MUST stop delivering events and release the associated resources.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#lifecycle + - check: sep-3415-stream-exempt-from-concurrency-cap + text: 'Server SDKs MUST exempt `events/stream` from any general cap on concurrent requests, because each push subscription is a long-lived request that does not complete until it is cancelled.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#lifecycle + - check: sep-3415-stream-carries-only-event-notifications + text: "The stream carries only this subscription's `notifications/events/*` messages. It is not a general server-to-client channel." + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#push-based-delivery + + # === Webhook: subscribing === + - check: sep-3415-subscribe-webhook-only + text: '`events/subscribe` is used only for webhook delivery. Poll and push do not need it.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscribing-eventssubscribe + - check: sep-3415-subscribe-secret-required + text: '`delivery.secret` is REQUIRED. The client supplies the HMAC signing secret, and the server never generates one.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscribing-eventssubscribe + - check: sep-3415-subscribe-secret-format + text: 'The value MUST be a Standard Webhooks symmetric secret: the literal prefix `whsec_` followed by the base64 encoding of 24 to 64 random bytes.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscribing-eventssubscribe + - check: sep-3415-subscribe-secret-rejected + text: 'The value MUST be a Standard Webhooks symmetric secret: the literal prefix `whsec_` followed by the base64 encoding of 24 to 64 random bytes. Servers MUST reject other values with `InvalidParams`.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscribing-eventssubscribe + - check: sep-3415-subscribe-url-https-required + text: 'Callback URLs MUST use `https://`.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + - check: sep-3415-subscribe-url-non-https-rejected + text: 'Servers MUST reject `events/subscribe` with a non-`https` `delivery.url` (`InvalidParams`).' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + - check: sep-3415-subscribe-auth-required + text: '`events/subscribe` and `events/unsubscribe` MUST be called with an authenticated principal. Servers MUST reject calls without an authorized principal with `Forbidden`.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscription-identity + - check: sep-3415-subscribe-key-composition + text: 'The subscription key is `(principal, delivery.url, name, arguments)`.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscription-identity + - check: sep-3415-subscribe-key-immutable + text: "All four components are immutable for the subscription's lifetime. A subscribe call with a different value for any of them addresses a different subscription." + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscription-identity + - check: sep-3415-subscribe-idempotent-upsert + text: "`events/subscribe` is idempotent within the caller's subscription scope. If a subscription with the same scoped key exists, the server resets the TTL and updates the mutable fields in place." + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscribing-eventssubscribe + - check: sep-3415-subscribe-id-derived + text: 'The server computes a deterministic `id` over the key, for example a truncated SHA-256 of the canonical key serialization, and returns it in the subscribe response.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscription-identity + - check: sep-3415-subscribe-id-not-an-input + text: "A caller who learns another tenant's derived `id` gains nothing, because `id` is not accepted as input to any method." + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscription-identity + - check: sep-3415-subscribe-refresh-replaces-secret + text: 'The client supplies a new `delivery.secret` on a refresh call, and the idempotent upsert replaces it.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + - check: sep-3415-subscribe-refresh-reactivates + text: 'A later successful refresh reactivates the subscription (sets `active: true`), and the server resumes retrying pending events.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-event-delivery + - check: sep-3415-subscribe-response-cursor + text: "It computes a safe-to-persist watermark, includes it in each delivered payload (see [Webhook event delivery](#webhook-event-delivery)), and returns the same watermark in each subscribe or refresh response, so the client's cursor advances during quiet periods." + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscribing-eventssubscribe + - check: sep-3415-subscribe-response-truncated + text: 'Webhook: The subscribe or refresh response: `{id, refreshBefore, cursor:, truncated:true, ...}`. For a gap detected between refreshes, the server POSTs a `{"type":"gap","cursor":""}` control envelope, so that the client learns of it without waiting for the next refresh.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#cursor-lifecycle + - check: sep-3415-subscribe-refresh-before-present + text: "`refreshBefore` (response, always present, nullable) is the server's authoritative grant: an ISO 8601 timestamp for when the subscription expires, or `null` for no expiry." + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscribing-eventssubscribe + - check: sep-3415-subscribe-cross-tenant-isolation + text: 'Because the key includes `principal` and `delivery.url`, two tenants that subscribe to the same `(name, arguments)` get distinct subscriptions.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscription-identity + + # === Webhook: TTL negotiation === + - check: sep-3415-ttl-refresh-before-lte-suggestion + text: '`refreshBefore` (response) is the grant. It SHOULD be less than or equal to the suggestion.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscription-ttl + - check: sep-3415-ttl-no-rejection-path + text: 'Clamping in either direction announces itself. The client reads the granted `refreshBefore` and schedules its refresh loop from that, so a clamped grant is not an error, and there is no rejection path for TTL values.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscription-ttl + - check: sep-3415-ttl-null-only-when-requested + text: 'A server MUST NOT return `null` unless the client suggested `ttlMs: null`, because no expiry exceeds every finite suggestion.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscription-ttl + - check: sep-3415-ttl-omitted-means-default + text: 'Omitting the field means "server default". An explicit `ttlMs: null` requests a subscription with **no expiry**.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscription-ttl + - check: sep-3415-ttl-long-grant-retained + text: 'A server that grants long or no-expiry TTLs MUST retain subscriptions for the lifetime it granted, including across restarts.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscribing-eventssubscribe + - check: sep-3415-ttl-no-expiry-persisted + text: 'The server MUST persist no-expiry subscriptions across restarts, because a client that never refreshes will never detect or repair one that was silently dropped.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscription-ttl + - check: sep-3415-ttl-no-expiry-gc-terminated + text: 'The server MAY drop a no-expiry subscription after sustained delivery failure over a server-defined window, and SHOULD attempt a `terminated` envelope when it does.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscription-ttl + + # === Webhook: unsubscribing === + - check: sep-3415-unsubscribe-by-key + text: '`events/unsubscribe` resolves the subscription with the same key as `events/subscribe`: `(principal, delivery.url, name, arguments)`.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#unsubscribing-eventsunsubscribe + - check: sep-3415-unsubscribe-empty-result + text: 'The result is an empty `Result`.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#unsubscribing-eventsunsubscribe + - check: sep-3415-unsubscribe-unknown-not-found + text: '`NotFound` `-32023`: A referenced entity does not exist: an unknown event name, or no subscription that matches the key on `events/unsubscribe`. The method implies which. `data.kind` (`"event"` | `"subscription"`) MAY disambiguate.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#error-handling + + # === Webhook: delivery status === + - check: sep-3415-delivery-status-last-error-category + text: '`lastError` MUST be a server-generated category string, one of `connection_refused`, `timeout`, `tls_error`, `http_4xx`, `http_5xx` or `challenge_failed`. It MUST NOT include raw response bodies, headers or status lines from the endpoint, so that it cannot serve as a response oracle for attacker-chosen URLs.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-delivery-status + + # === Webhook: delivery mechanics === + - check: sep-3415-delivery-post-json + text: 'Deliveries are HTTP `POST` only. - **Content-Type.** `application/json`.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + - check: sep-3415-delivery-standard-webhooks-headers + text: 'Every delivery MUST include `webhook-id`, `webhook-timestamp` and `webhook-signature`.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + - check: sep-3415-delivery-subscription-id-header + text: 'Deliveries MUST also include `X-MCP-Subscription-Id` (the subscription `id`), so that the receiver can select the correct secret without parsing the body.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + - check: sep-3415-delivery-signature-formula + text: 'The signature is `HMAC-SHA256(secret, webhook-id + "." + webhook-timestamp + "." + body)`, encoded as base64 with a `v1,` prefix. `body` is the raw HTTP request body bytes exactly as received, before any parsing or re-serialization. The `webhook-id + "." + webhook-timestamp + "."` prefix is UTF-8. `secret` is the base64-decoded bytes of the value after the `whsec_` prefix.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + - check: sep-3415-delivery-retry-regenerates-signature + text: "Each retry attempt MUST regenerate the timestamp and signature, so that retries are not rejected by the receiver's freshness window." + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + - check: sep-3415-delivery-dual-sign-on-rotation + text: 'The server SHOULD dual-sign deliveries with both the old and new secrets for a short grace window, so that in-flight deliveries verify under either.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + - check: sep-3415-delivery-body-size + text: 'Servers SHOULD keep delivery bodies at or under 256 KiB, consistent with [Payload minimality](#payload-minimality).' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + - check: sep-3415-delivery-413-non-retryable + text: 'Servers MUST treat `413` as a non-retryable failure for that event.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + - check: sep-3415-delivery-410-non-retryable + text: 'The server MUST treat `410` as a non-retryable failure for that delivery (like `413`, see [Delivery profile](#webhook-security)), without affecting the subscription itself.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-event-delivery + - check: sep-3415-delivery-retries-bounded + text: 'Servers SHOULD cap both the attempt count and the elapsed retry window, for example 3 to 5 attempts spread over no more than 10 to 15 minutes, and SHOULD NOT retry indefinitely.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-event-delivery + + # === Webhook: SSRF and endpoint verification === + - check: sep-3415-ssrf-validate-callback-url + text: 'The server MUST validate callback URLs.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + - check: sep-3415-ssrf-reject-non-routable + text: 'Servers SHOULD reject URLs whose resolved IP is not globally routable per the [IANA IPv4](https://www.iana.org/assignments/iana-ipv4-special-registry/) and [IPv6](https://www.iana.org/assignments/iana-ipv6-special-registry/) Special-Purpose Address Registries, unless they are explicitly configured to allow them.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + - check: sep-3415-ssrf-validate-at-delivery-time + text: 'To prevent DNS rebinding, this validation MUST be performed at delivery time, and not only at subscribe time. The server resolves the hostname, checks the resolved IP against the blocklist, and connects directly to that validated IP, sending the original hostname in the `Host` header and TLS SNI, so that the address cannot change between the check and the connection.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + - check: sep-3415-ssrf-no-redirects + text: 'Webhook delivery requests MUST NOT follow HTTP redirects, because a redirect can target an internal address that bypasses the blocklist.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + - check: sep-3415-verification-required-before-delivery + text: "A server therefore MUST NOT begin delivering to a callback URL until the endpoint's intent to receive deliveries is confirmed, by **one of**:" + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + - check: sep-3415-verification-challenge-echo + text: 'Before activating, the server POSTs a `verification` control envelope that carries a single-use, short-lived `challenge` nonce, signed and headed like any delivery. The endpoint proves intent by echoing the nonce in a `2xx` body (`{"challenge":""}`), which the server compares in constant time.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + - check: sep-3415-verification-failure-error + text: 'A reachable endpoint that fails to echo the challenge yields `CallbackEndpointError` with `data.reason: "challenge_failed"`. An unreachable one yields the same error with the relevant connection-failure category (`connection_refused`, `timeout` or `tls_error`).' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + - check: sep-3415-verification-cached-per-principal-url + text: "Verification is cached per `(principal, url)`. A successful handshake, allowlist hit or well-known match covers that principal's subscriptions to that URL across refreshes and `arguments`. Varying `arguments` therefore cannot multiply verification POSTs at a victim, and one principal's verification never waives the challenge for another." + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + - check: sep-3415-verification-persisted-for-no-expiry + text: 'A server that persists no-expiry subscriptions across restarts MUST persist their verification status alongside them.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + - check: sep-3415-verification-uses-ssrf-hardened-path + text: 'The verification POST MUST use the same SSRF-hardened path as deliveries (delivery-time IP validation, no redirects)' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + - check: sep-3415-verification-no-raw-endpoint-responses + text: 'Failures surface only through the `lastError` category `challenge_failed`, and never through raw endpoint responses.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + - check: sep-3415-server-identity-key-discovery + text: 'The verifying public key MUST be discovered from an origin the client already authenticates, and never from the challenge or delivery body, which the attacker controls and which would make this trust-on-first-use.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + + # === Webhook: control envelopes === + - check: sep-3415-envelope-type-discriminator + text: 'A body with a top-level `type` field is a control envelope, and a body without one is an `EventOccurrence`.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#non-event-webhook-bodies + - check: sep-3415-envelope-signed-like-deliveries + text: 'Control envelopes are signed and carry the same headers as event deliveries (the Standard Webhooks headers and `X-MCP-Subscription-Id`).' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#non-event-webhook-bodies + - check: sep-3415-envelope-webhook-id-format + text: '`webhook-id` for a control envelope is a per-message identifier of the form `msg__`, so that receivers can deduplicate retries.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#non-event-webhook-bodies + - check: sep-3415-envelope-gap + text: '`gap` `{"type":"gap","cursor":""}`: A gap is detected between refreshes (see [Cursor Lifecycle](#cursor-lifecycle)). The client persists `cursor` and treats it as `truncated: true`.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#non-event-webhook-bodies + - check: sep-3415-envelope-terminated + text: '`terminated` `{"type":"terminated","error":{"code":...,"message":...,"data":...}}`: The subscription has ended, for example because authorization was revoked. The subscription no longer exists on the server.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#non-event-webhook-bodies + + # === Authorization === + - check: sep-3415-authz-subscribe-time + text: 'When the caller is authenticated, the server MUST verify that the principal has permission to subscribe to the requested event type with the given arguments.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#authorization + - check: sep-3415-authz-delivery-time-reverify + text: 'The server SHOULD periodically re-verify permissions.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#authorization + + # === Payload handling (server side) === + - check: sep-3415-payload-minimality + text: 'Servers SHOULD keep event payloads minimal: enough to identify and triage the event, and not the full content.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#payload-minimality + + # ========================================================================== + # Excluded rows. Each carries an RFC 2119 keyword in the source but is not a + # server wire obligation, so no server-side scenario can emit a check for it. + # Kept here so the keyword sweep stays auditable against the document. + # ========================================================================== + + # --- Client / host obligations --- + - text: 'The client SHOULD call `events/list` again to refresh its registry of event types.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#dynamic-event-types + excluded: client-side obligation; needs a client-conformance scenario set + - text: 'When it is `true`, the client SHOULD poll again immediately with the updated cursor, ignoring `nextPollMs`, to drain the backlog.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#response-1 + excluded: client-side obligation; needs a client-conformance scenario set + - text: 'Clients SHOULD apply a configurable floor (default 1000 ms) so that a misbehaving server cannot induce a tight loop.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#response-1 + excluded: client-side obligation; needs a client-conformance scenario set + - text: 'Clients SHOULD NOT apply their default request timeout to `events/stream`: the heartbeat is the liveness signal. A client that has received neither an event nor a heartbeat for more than twice the heartbeat interval SHOULD treat the stream as dead and reconnect with its cursor.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#lifecycle + excluded: client-side obligation; needs a client-conformance scenario set + - text: 'Unless it was granted no expiry, the client MUST call `events/subscribe` again with the same subscription key before `refreshBefore` to keep the subscription alive.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscribing-eventssubscribe + excluded: client-side obligation; needs a client-conformance scenario set + - text: 'Even with no expiry, clients SHOULD still call `events/subscribe` and `events/list` occasionally.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscription-ttl + excluded: client-side obligation; needs a client-conformance scenario set + - text: 'A client that receives `cursor: null` MUST NOT attempt to persist or replay from it. On reconnect or resubscribe it sends `cursor: null` (start from now)' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#cursor-lifecycle + excluded: client-side obligation; needs a client-conformance scenario set + - text: '`truncated: true` always implies a possible gap. Clients SHOULD treat it as one, for example by fetching authoritative state again with tools if it matters, and SHOULD persist the fresh cursor.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#cursor-lifecycle + excluded: client-side obligation; needs a client-conformance scenario set + - text: 'Client SDKs SHOULD generate the secret for the application, and SHOULD NOT expose an interface that encourages hand-picked values.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscribing-eventssubscribe + excluded: client-SDK guidance; not observable from the server under test + - text: 'Clients SHOULD sanitize or sandbox event payloads before presenting them to an LLM, following the security model the specification already defines for tool results.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#event-payloads-are-untrusted-data + excluded: host obligation; not observable from the server under test + - text: 'Clients SHOULD support policy evaluation between event receipt and action execution.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#governance + excluded: host obligation (Enterprise Governance); not observable from the server under test + - text: 'The client SDK SHOULD remove the subscription and notify the application.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscription-termination + excluded: client-SDK guidance; not observable from the server under test + + # --- Receiver (callback endpoint) obligations --- + - text: 'The receiver MUST verify the signature before processing, SHOULD reject deliveries whose `webhook-timestamp` is more than 5 minutes old, and SHOULD deduplicate on `webhook-id`.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-event-delivery + excluded: receiver obligation; the harness implements this rather than grading it + - text: 'Receivers MUST compute the HMAC over the raw body, and never over a re-serialized JSON object.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + excluded: receiver obligation; the harness implements this rather than grading it + - text: 'The endpoint MUST forward `gap` and `terminated` envelopes to the consuming client by the same channel it uses to forward events.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#non-event-webhook-bodies + excluded: receiver obligation; the harness implements this rather than grading it + - text: 'The endpoint SHOULD NOT return `2xx` until the event has been durably persisted or forwarded.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-event-delivery + excluded: receiver obligation; the harness implements this rather than grading it + - text: 'A receiver that gets a delivery for an `id` it has not yet been told to route, for example because the subscribe response has not propagated to the gateway, SHOULD return a retryable status (`503` or `425 Too Early`).' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-event-delivery + excluded: receiver obligation; the harness implements this rather than grading it + + # --- Server-SDK implementation guidance (not wire-observable) --- + - text: 'The SDK SHOULD enable poll by default, so that it is available unless the author opts out.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#server-sdk-guidance + excluded: server-SDK guidance; indistinguishable on the wire from an author opting in + - text: 'SDKs SHOULD refuse a no-expiry cap unless the author has wired up durable storage.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#server-sdk-guidance + excluded: server-SDK guidance; a construction-time policy with no wire signal + - text: 'SDKs SHOULD provide an in-memory ring buffer that retains a bounded window of emitted events per event type, configurable by time, by count or by both.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#server-sdk-guidance + excluded: server-SDK guidance; an internal structure with no wire signal + - text: 'Servers SHOULD enforce the subscription limit (`ResourceExhausted`) before they invoke `on_subscribe`, so that a rejected subscription never provisions upstream resources.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#server-sdk-guidance + excluded: server-SDK ordering guidance; the harness cannot observe hook invocation order + - text: "The lease window is configurable in the SDK, and SHOULD default to a small multiple of the server's typical `nextPollMs`, so that a well-behaved client never lapses between polls:" + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#server-sdk-guidance + excluded: server-SDK guidance; an internal lease table with no wire signal + - text: 'For servers that wrap upstreams that _do_ offer a durable cursor (Gmail `historyId`, Kubernetes `resourceVersion`, Stripe `/v1/events`, a Kafka offset), authors SHOULD prefer a check function that queries the upstream directly.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#server-sdk-guidance + excluded: server-authoring guidance; a design recommendation with no wire signal + - text: 'On HTTP/1.1 each stream is a TCP connection, so SDKs SHOULD prefer HTTP/2 when many subscriptions are active.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#how-each-mode-works + excluded: transport guidance; not a protocol conformance obligation + + # --- Deployment profile advisories --- + - text: 'A WAF MAY drop requests that are missing any of them as a cheap pre-filter.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + excluded: deployment advisory addressed to operators, not to an implementation + + # --- Pure MAY / OPTIONAL (handoff disposition: no row) --- + - text: 'A server MAY return `cursor: null` in any delivery (a poll result, a push `notifications/events/event` or `notifications/events/active`, a webhook payload) when the event type does not support replay.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#cursor-lifecycle + excluded: pure MAY; kept as context for sep-3415-cursor-consistency, which carries the testable half + - text: 'The one sanctioned exception is a server-side floor: a server MAY clamp an impractically short suggestion up to its minimum TTL, to protect itself from refresh storms.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#subscription-ttl + excluded: pure MAY; the testable half is sep-3415-ttl-refresh-before-lte-suggestion, which this sentence excepts + - text: 'The header MAY contain several space-delimited signatures (`v1, v1,`) during secret rotation. The receiver accepts the delivery if any of them verifies.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-security + excluded: pure MAY, and the obligation it creates falls on the receiver + - text: 'After repeated failures, at a server-defined threshold, the server MAY suspend delivery (`deliveryStatus.active: false`).' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-event-delivery + excluded: pure MAY; suspension is observable only through the OPTIONAL deliveryStatus object + - text: '`deliveryStatus` is OPTIONAL, and servers MAY omit it entirely.' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#webhook-delivery-status + excluded: OPTIONAL by its own wording; a server omitting deliveryStatus entirely is conformant + - text: "Servers SHOULD therefore evolve an event type's `inputSchema` and `payloadSchema` additively for the lifetime of its `name`: - New optional fields MAY be added. - Existing fields SHOULD NOT be removed, renamed or retyped. - Enums SHOULD NOT be narrowed. - `inputSchema` SHOULD NOT be tightened so that previously accepted `arguments` become invalid." + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#response + excluded: server policy over a catalog's lifetime; a harness can force a change but cannot observe whether the server would have chosen it, so the row has no verdict (was sep-3415-schema-evolution-additive, moved to excluded in panyam/mcpkit#1465) + - text: 'A breaking change SHOULD instead be published under a new event name and served alongside the old one for a migration period, after which the old name is removed' + url: https://github.com/modelcontextprotocol/modelcontextprotocol/blob/c47fd2451c91ad11708dd498ddee060a1d73aedc/seps/0000-events-extension.md#response + excluded: server policy over a catalog's lifetime; a harness can force a change but cannot observe whether the server would have chosen it, so the row has no verdict (was sep-3415-breaking-change-new-name, moved to excluded in panyam/mcpkit#1465) diff --git a/src/types.ts b/src/types.ts index ebe75a27..20c058b8 100644 --- a/src/types.ts +++ b/src/types.ts @@ -103,7 +103,13 @@ export const EXTENSION_IDS = [ 'io.modelcontextprotocol/auth/dpop', 'io.modelcontextprotocol/auth/wif', 'io.modelcontextprotocol/tasks', - 'io.modelcontextprotocol/skills' + 'io.modelcontextprotocol/skills', + // MCP Events. This id does double duty: it keeps the Events scenarios off the + // `--spec-version` timeline (see `matchesSpecVersion`), and it is the key the + // capability lives under in `capabilities.extensions`. Extension negotiation + // arrived in 2026-07-28, so no earlier protocol version can declare it. + // See src/scenarios/server/events/helpers.ts. + 'io.modelcontextprotocol/events' ] as const; export type ExtensionId = (typeof EXTENSION_IDS)[number];