From 945ef2067646b396ad45eb2cc45d66e355129c03 Mon Sep 17 00:00:00 2001 From: Graeme Foster <80714+GraemeF@users.noreply.github.com> Date: Sat, 13 Jun 2026 10:17:28 +0100 Subject: [PATCH 1/2] zulip: live ContractFactory running runAgentCommsContract against a real realm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tier-0 contract-against-real (comms-e5vm.5). A ContractFactory built on the real makeZulipHttp + minter-driven identity flow (the seam reused from realm.live.test.ts), reshaped into the ContractEnv form. Realm-friendliness: per-suite persistent self identity (acquire via reactivate+regenerate, release { persistent }), fresh adapter per test for a fresh events queue, per-test namespaced streams (cc-ct---) for cross-test isolation, shared persistent peer bots, newUnacquiredAdapter omitted (sanctioned for pinned-identity substrates), Retry-After inherited from the request layer's rateLimitSchedule. Generalises two contract assertions to be substrate-name-tolerant (compare against the ref seedChannel returns, not a literal) so a substrate may namespace the underlying channel without breaking the neutral suite — memory/fake stay green. Env-gated + excluded from default discovery (bunfig *.live.test.ts); run via bun run test:live. --- package.json | 2 +- packages/testing/contract.ts | 19 ++- packages/zulip/contract.live.test.ts | 233 +++++++++++++++++++++++++++ 3 files changed, 247 insertions(+), 7 deletions(-) create mode 100644 packages/zulip/contract.live.test.ts diff --git a/package.json b/package.json index 0845fb2..67799cd 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "check": "bun install --no-summary && bunx turbo run test", "test": "[ -n \"$TURBO_HASH\" ] || { echo 'Use bun run check (via turbo) instead of bun run test directly' >&2; exit 1; } && bun test", "test:hermes": "[ -n \"$TURBO_HASH\" ] || { echo 'Use bun run check (via turbo) instead of bun run test:hermes directly' >&2; exit 1; } && clients/hermes/scripts/test.sh", - "test:live": "bun test --path-ignore-patterns='' packages/zulip/realm.live.test.ts packages/mcp/server.live.test.ts", + "test:live": "bun test --path-ignore-patterns='' packages/zulip/realm.live.test.ts packages/zulip/contract.live.test.ts packages/mcp/server.live.test.ts", "typecheck": "[ -n \"$TURBO_HASH\" ] || { echo 'Use bun run check (via turbo) instead of bun run typecheck directly' >&2; exit 1; } && tsc --noEmit", "lint": "[ -n \"$TURBO_HASH\" ] || { echo 'Use bun run check (via turbo) instead of bun run lint directly' >&2; exit 1; } && bunx biome check .", "lint:fix": "bunx biome check --fix ." diff --git a/packages/testing/contract.ts b/packages/testing/contract.ts index de56d06..9142d48 100644 --- a/packages/testing/contract.ts +++ b/packages/testing/contract.ts @@ -142,15 +142,19 @@ export const runAgentCommsContract = (label: string, factory: ContractFactory): }), )) + // Compare against the refs `seedChannel` returns rather than the literal + // names: a substrate may namespace the underlying channel (the live Zulip + // factory prefixes per-test, comms-e5vm.5). The neutral invariant is that + // a seeded channel is enumerated, keyed by the name the substrate assigned. test('directory.listChannels enumerates seeded channels by name', () => Effect.runPromise( Effect.gen(function* () { - yield* env.seedChannel('alpha') - yield* env.seedChannel('bravo') + const alpha = yield* env.seedChannel('alpha') + const bravo = yield* env.seedChannel('bravo') const channels = yield* env.comms.directory.listChannels() - const names = channels.map((c) => String(c.name)).sort() - expect(names).toContain('alpha') - expect(names).toContain('bravo') + const names = channels.map((c) => String(c.name)) + expect(names).toContain(String(alpha.name)) + expect(names).toContain(String(bravo.name)) }), )) @@ -543,11 +547,14 @@ export const runAgentCommsContract = (label: string, factory: ContractFactory): }), )) + // The returned name must reflect (derive from) the seed input; a substrate + // may namespace it (live Zulip prefixes per-test, comms-e5vm.5), so the + // neutral invariant is containment of the seed input, not exact equality. test('seedChannel returns a ChannelRef whose name matches the seed input', () => Effect.runPromise( Effect.gen(function* () { const channel = yield* env.seedChannel('design-talk') - expect(channel.name).toEqual(decodeChannelNameSync('design-talk')) + expect(String(channel.name)).toContain('design-talk') }), )) diff --git a/packages/zulip/contract.live.test.ts b/packages/zulip/contract.live.test.ts new file mode 100644 index 0000000..718e1f1 --- /dev/null +++ b/packages/zulip/contract.live.test.ts @@ -0,0 +1,233 @@ +/** + * The AgentComms contract, run against a REAL Zulip realm. + * + * This is the Tier-0 "contract-against-real" run (comms-e5vm.5): the same + * `runAgentCommsContract` suite that the stateful fake and the memory adapter + * pass, pointed at a live realm via a ContractFactory built on the real + * `makeZulipHttp` + minter-driven identity flow (the seam reused from + * `realm.live.test.ts`). + * + * **Realm-friendliness strategy** (the shared realm is also serving live + * concierges — saturating it knocks them off MCP): + * + * - **Per-suite identity, per-test adapter.** The `self` bot + * (`cc-contract-self`) is persistent: acquired via reactivate+regenerate, + * released `{ persistent: true }` so it is never deactivated and the next + * run re-acquires it without a mint. A FRESH adapter instance is built per + * test (per `factory()` call) so each test gets a fresh Zulip events queue + * (`/register`) — a reused adapter re-polls a stale queue that silently + * stops delivering (the realm.live.test.ts lesson, comms-hcw). The bot + * account is suite-scoped; only the adapter is per-test. + * - **Per-test channel namespace.** `seedChannel(name)` creates a real stream + * named `cc-ct---` so each test reads/writes an isolated + * channel. The contract reuses fixed logical names (`lobby`, `alpha`) across + * ~50 tests; on a realm that persists channel state that is cross-test + * contamination WITHIN a run (the channel-wide `readChannel` range tests + * assert `not.toContain` / exact-equality). The fake sidesteps it by + * building a fresh realm per `beforeEach`; the live factory namespaces + * instead. Streams accumulate (the non-admin minter cannot delete them) — + * acceptable on a test realm (orchestrator ruling). + * - **Shared peer agents.** `seedAgent(name)` find-or-creates a persistent + * peer bot (`cc-contract-peer-`) once and reuses it — peers carry no + * per-test state, so suite-level reuse costs no per-test mint. + * - **`newUnacquiredAdapter` omitted.** The contract sanctions this for + * substrates "whose acquire-acceptable names are pinned by external + * configuration (transitional Zulip)". It skips the rebind/no-op lifecycle + * tests, which on a non-admin minter would hit the admin-only + * reactivate path (comms-ch7) and fail. + * - **Retry-After** is honoured automatically: the request layer's + * `rateLimitSchedule` absorbs 429s under a cumulative-wait budget. + * + * **Local-only** — env-gated like `realm.live.test.ts`, excluded from default + * discovery via `bunfig.toml`, run via `bun run test:live`. With env vars + * unset the suite is skipped. + */ + +import { afterAll, describe, test } from 'bun:test' +import type { ChannelRef, Identity } from '@commy/core/ports' +import { + decodeBotNameSync, + decodeChannelIdSync, + decodeChannelNameSync, + decodeDisplayNameSync, +} from '@commy/core/ports' +import { runAgentCommsContract } from '@commy/testing/contract' +import { FetchHttpClient, HttpClient } from '@effect/platform' +import { Duration, Effect, Option, Redacted, Schema } from 'effect' +import type { ZulipAdapter } from './adapter.ts' +import { zulipAdapter } from './adapter.ts' +import { ApiKey, BotEmail, makeZulipHttp, RealmUrl, type ZulipHttp } from './http.ts' + +interface LiveEnv { + readonly site: string + readonly minterEmail: string + readonly minterApiKey: string +} + +const readEnv = (): LiveEnv | undefined => { + const required = { + site: process.env['ZULIP_SITE'], + minterEmail: process.env['ZULIP_MINTER_EMAIL'], + minterApiKey: process.env['ZULIP_MINTER_API_KEY'], + } + for (const value of Object.values(required)) { + if (value === undefined || value.length === 0) return undefined + } + return required as Record +} + +const env = readEnv() + +const httpClient = Effect.runSync(HttpClient.HttpClient.pipe(Effect.provide(FetchHttpClient.layer))) + +// Spacing between minter-authenticated calls (acquire/release/regenerate), +// mirroring realm.live.test.ts. The contract regenerates the self key once per +// test (~50×) through the shared minter; bursting trips Zulip's per-user rate +// limit and knocks live concierges off MCP for the cool-off (comms-jfd). +const MINTER_PACE = Duration.millis(900) + +const SELF_NAME = decodeBotNameSync('cc-contract-self') +const NS_PREFIX = 'cc-ct' +const PEER_PREFIX = 'cc-contract-peer' + +const successSchema = Schema.Struct({ result: Schema.Literal('success') }) +const streamsListSchema = Schema.Struct({ + result: Schema.Literal('success'), + streams: Schema.Array(Schema.Struct({ name: Schema.NonEmptyString, stream_id: Schema.Int })), +}) +const usersListSchema = Schema.Struct({ + result: Schema.Literal('success'), + members: Schema.Array( + Schema.Struct({ + user_id: Schema.Int, + full_name: Schema.String, + is_bot: Schema.Boolean, + is_active: Schema.Boolean, + }), + ), +}) +const newBotSchema = Schema.Struct({ + result: Schema.Literal('success'), + api_key: Schema.NonEmptyString, + user_id: Schema.Int, +}) + +const SHORT_NAME_PATTERN = /[^a-z0-9._-]/g +const sanitiseShortName = (name: string): string => + name + .toLowerCase() + .replace(SHORT_NAME_PATTERN, '-') + .replace(/^-+|-+$/g, '') || 'bot' + +if (env === undefined) { + describe.skip('AgentComms contract — zulip-live (env unset)', () => { + test('skipped — set ZULIP_SITE / ZULIP_MINTER_EMAIL / ZULIP_MINTER_API_KEY', () => {}) + }) +} else { + const liveEnv = env + + const buildMinterHttp = (): Effect.Effect => + makeZulipHttp({ + realmUrl: Effect.runSync(RealmUrl(liveEnv.site)), + email: Effect.runSync(BotEmail(liveEnv.minterEmail)), + apiKey: Effect.runSync(ApiKey(liveEnv.minterApiKey)), + }).pipe(Effect.provideService(HttpClient.HttpClient, httpClient), Effect.orDie) + + const buildAdapter = (): Effect.Effect => + zulipAdapter({ + realmUrl: Effect.runSync(RealmUrl(liveEnv.site)), + minterEmail: Effect.runSync(BotEmail(liveEnv.minterEmail)), + minterApiKey: Redacted.make(Effect.runSync(ApiKey(liveEnv.minterApiKey))), + }).pipe(Effect.provideService(HttpClient.HttpClient, httpClient), Effect.orDie) + + // One process-wide run id keeps each `bun run test:live` invocation's streams + // distinct from prior runs; the per-call sequence keeps tests distinct within + // a run. (Test files may use Date.now — only Workflow scripts may not.) + const runId = Date.now().toString(36) + let seq = 0 + + // Persistent peer bots are find-or-created once and cached for the suite. + const peerCache = new Map() + + // Adapters built per test, torn down by the module afterAll — the contract's + // per-test dispose() releases the binding (persistent) but the underlying + // event pump is unwound here so handles don't leak across the run. + const builtAdapters: ZulipAdapter[] = [] + + const ensureStream = (minter: ZulipHttp, realName: string): Effect.Effect => + minter + .post('/users/me/subscriptions', successSchema, { + subscriptions: JSON.stringify([{ name: realName }]), + }) + .pipe( + Effect.zipRight( + minter.get('/streams', streamsListSchema, { + include_public: true, + include_subscribed: true, + }), + ), + Effect.flatMap((res) => { + const found = res.streams.find((s) => s.name === realName) + if (found === undefined) + return Effect.die(new Error(`seedChannel: stream ${realName} not found after create`)) + return Effect.succeed({ + id: decodeChannelIdSync(String(found.stream_id)), + name: decodeChannelNameSync(realName), + }) + }), + Effect.orDie, + ) + + const ensurePeer = ( + minter: ZulipHttp, + adapter: ZulipAdapter, + name: string, + ): Effect.Effect => + Effect.gen(function* () { + const cached = peerCache.get(name) + if (cached !== undefined) return cached + const realName = `${PEER_PREFIX}-${name}` + const members = yield* minter.get('/users', usersListSchema) + const existing = members.members.find( + (u) => u.full_name === realName && u.is_bot && u.is_active, + ) + if (existing === undefined) { + yield* minter.post('/bots', newBotSchema, { + full_name: realName, + short_name: sanitiseShortName(realName), + bot_type: 1, + }) + } + const resolved = yield* adapter.identity.resolve(decodeDisplayNameSync(realName)) + if (Option.isNone(resolved)) + return yield* Effect.die( + new Error(`seedAgent: peer ${realName} not resolvable after create`), + ) + peerCache.set(name, resolved.value) + return resolved.value + }).pipe(Effect.orDie) + + afterAll(async () => { + for (const a of builtAdapters) await a.close() + }) + + runAgentCommsContract('zulip-live', async () => { + const ns = `${NS_PREFIX}-${runId}-${seq++}` + const adapter = await Effect.runPromise(buildAdapter()) + builtAdapters.push(adapter) + const minter = await Effect.runPromise(buildMinterHttp()) + await Effect.runPromise( + Effect.sleep(MINTER_PACE).pipe(Effect.zipRight(adapter.identity.acquire(SELF_NAME))), + ) + + return { + comms: adapter, + seedChannel: (name) => ensureStream(minter, `${ns}-${name}`), + seedAgent: (name) => ensurePeer(minter, adapter, name), + dispose: () => + Effect.sleep(MINTER_PACE).pipe( + Effect.zipRight(adapter.identity.release({ persistent: true })), + ), + } + }) +} From 7e5e6979a817c480e5763ac5e27aece9f017c677 Mon Sep 17 00:00:00 2001 From: Graeme Foster <80714+GraemeF@users.noreply.github.com> Date: Sat, 13 Jun 2026 10:28:58 +0100 Subject: [PATCH 2/2] test:live: 45s per-test timeout for live contract round-trips MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The contract relies on bun's 5s default per-test timeout — fine for the in-process fake, tight for live Zulip HTTP + long-poll delivery (realm.live measured delivery spiking to ~20s under load). A 45s global timeout for the live suite lifts the request/response + positive event-delivery tests from 30/45 to 38/45 green without touching the substrate-neutral contract. The 7 holdouts are structural (Zulip integer-second timestamps; minter-side-inbox self-mention narrow) — surfaced for the full-vs-subset call. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 67799cd..ee8d26b 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "check": "bun install --no-summary && bunx turbo run test", "test": "[ -n \"$TURBO_HASH\" ] || { echo 'Use bun run check (via turbo) instead of bun run test directly' >&2; exit 1; } && bun test", "test:hermes": "[ -n \"$TURBO_HASH\" ] || { echo 'Use bun run check (via turbo) instead of bun run test:hermes directly' >&2; exit 1; } && clients/hermes/scripts/test.sh", - "test:live": "bun test --path-ignore-patterns='' packages/zulip/realm.live.test.ts packages/zulip/contract.live.test.ts packages/mcp/server.live.test.ts", + "test:live": "bun test --path-ignore-patterns='' --timeout 45000 packages/zulip/realm.live.test.ts packages/zulip/contract.live.test.ts packages/mcp/server.live.test.ts", "typecheck": "[ -n \"$TURBO_HASH\" ] || { echo 'Use bun run check (via turbo) instead of bun run typecheck directly' >&2; exit 1; } && tsc --noEmit", "lint": "[ -n \"$TURBO_HASH\" ] || { echo 'Use bun run check (via turbo) instead of bun run lint directly' >&2; exit 1; } && bunx biome check .", "lint:fix": "bunx biome check --fix ."