From 468ebd9d1433b0f061310d8bf998a5cdd25b10f3 Mon Sep 17 00:00:00 2001 From: danielxxomg Date: Thu, 2 Jul 2026 13:03:50 -0500 Subject: [PATCH] feat(model): add retry/backoff with error classification and reconnect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace naive exponential backoff with production-grade retry engine: - Error classification: 18 non-retryable patterns (auth/quota/validation) and 24 retryable patterns (network/server/timeout) via extractMessage() that handles Error, string, and SSE plain-object shapes - Fixed backoff schedule [1s, 2.5s, 5s] with ±25% jitter replaces unbounded exponential 2^n growth - fetchWithRetry: retries 5xx/429 with backoff, fails fast on 4xx - fetchOnce: single fetch for mid-stream reconnects (no double counting) - streamWithReconnect: emittedContent tracking gates reconnect to prevent duplicate content generation; pendingReconnect state machine handles clean reconnection flow - shouldRetry: centralized decision — checks aborted, emittedContent, attempt count, and error classification - buildHttpError: parses JSON error bodies with model ID annotation - partialOutputError: clear error when reconnect is unsafe - wrapAsError: normalizes any error shape into Error with ccError/code --- src/model.ts | 415 +++++++++++++++++++++++++++++++++++++-- tests/helpers/mocks.ts | 68 +++++++ tests/unit/model.test.ts | 192 +++++++++++++++++- 3 files changed, 650 insertions(+), 25 deletions(-) diff --git a/src/model.ts b/src/model.ts index 8d0cbe7..a0f582e 100644 --- a/src/model.ts +++ b/src/model.ts @@ -6,6 +6,7 @@ import type { LanguageModelV3Content, LanguageModelV3Usage, LanguageModelV3FinishReason, + LanguageModelV3StreamPart, } from "@ai-sdk/provider" import { buildRequest } from "./convert.js" import { parseStreamEvents } from "./stream.js" @@ -14,6 +15,201 @@ const DEFAULT_BASE_URL = "https://api.commandcode.ai" // x-command-code-version must match the Command Code CLI version for API compatibility const CC_VERSION = "0.26.20" +// --- Retry config --- +// Max 3 retries after the initial attempt (4 total). Backoff schedule with +// jitter keeps reconnects short: ~1s, ~2.5s, ~5s (±25% jitter). +const MAX_RETRIES = 3 +const BACKOFF_SCHEDULE_MS = [1000, 2500, 5000] as const +const REQUEST_TIMEOUT_MS = 300_000 + +// --- error classification --- + +/** + * Extract a lowercased, searchable message from an error of any shape. + * Handles Error instances, strings, and the plain SSE error objects Command + * Code emits (e.g. { type: "server_error", message: "Network connection lost." }). + */ +function extractMessage(err: unknown): string { + if (err === null || err === undefined) return "" + if (err instanceof Error) return err.message.toLowerCase() + if (typeof err === "string") return err.toLowerCase() + if (typeof err === "object") { + const e = err as Record + const nested = e.error as Record | undefined + const parts: string[] = [] + for (const v of [e.message, nested?.message, e.msg, nested?.type, e.type, e.code]) { + if (typeof v === "string" && v) parts.push(v) + } + if (parts.length) return parts.join(" ").toLowerCase() + try { + return JSON.stringify(err).toLowerCase() + } catch { + return "" + } + } + return String(err).toLowerCase() +} + +// Non-transient failures: never retry these. Matched conservatively (specific +// phrases) so genuine transient errors are never misclassified as permanent. +const NON_RETRYABLE_PATTERNS = [ + "insufficient credit", + "insufficient_credit", + "model_not_in_plan", + "model not in plan", + "not_in_plan", + "not in plan", + "usage limit", + "usage_limit", + "exceeded your", + "quota exceeded", + "unauthorized", + "forbidden", + "invalid api key", + "invalid_api_key", + "authentication", + "auth_error", + "permission_denied", + "validation_error", + "bad request", + "not found", +] + +// Transient failures: safe to retry when no content has been emitted yet. +const RETRYABLE_PATTERNS = [ + "network connection lost", + "connection lost", + "connection reset", + "connection refused", + "connection timeout", + "server_error", + "server error", + "internal server error", + "internal error", + "aborted", + "aborterror", + "abort_error", + "fetch failed", + "fetchfailed", + "econnreset", + "econnrefused", + "etimedout", + "socket hang up", + "terminated", + "bad gateway", + "service unavailable", + "gateway timeout", + "downstream", + "temporarily unavailable", +] + +function isNonRetryableError(err: unknown): boolean { + const msg = extractMessage(err) + if (!msg) return false + return NON_RETRYABLE_PATTERNS.some((p) => msg.includes(p)) +} + +function isRetryableError(err: unknown): boolean { + if (isNonRetryableError(err)) return false + const msg = extractMessage(err) + if (!msg) return false + return RETRYABLE_PATTERNS.some((p) => msg.includes(p)) +} + +function isRetryableStatus(status: number): boolean { + return status === 429 || status >= 500 +} + +function backoffDelay(attempt: number): number { + const base = BACKOFF_SCHEDULE_MS[Math.min(attempt, BACKOFF_SCHEDULE_MS.length - 1)] + // jitter ±25% to avoid synchronized retry storms + return base * (0.75 + Math.random() * 0.5) +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +function describeError(err: unknown): string { + if (err instanceof Error) return err.message + if (typeof err === "string") return err + try { + return JSON.stringify(err) + } catch { + return String(err) + } +} + +function wrapAsError(err: unknown): Error { + if (err instanceof Error) return err + if (typeof err === "string") return new Error(err) + const e = err as Record + const nested = e.error as Record | undefined + const message = + (typeof e.message === "string" && e.message) || + (typeof nested?.message === "string" && nested.message) || + (typeof e.msg === "string" && e.msg) || + (() => { + try { + return JSON.stringify(err) + } catch { + return "Unknown error" + } + })() + const type = + (typeof e.type === "string" && e.type) || + (typeof nested?.type === "string" && nested.type) || + undefined + const error = new Error(type ? `${type}: ${message}` : String(message)) + Object.assign(error, { ccError: err, ...(type ? { code: type } : {}) }) + return error +} + +function partialOutputError(original: unknown): Error { + const err = new Error( + `Command Code stream failed after partial output was already emitted; reconnect aborted to avoid duplicate content. Original error: ${describeError(original)}`, + ) + Object.assign(err, { partialOutput: true, ccError: original }) + return err +} + +/** + * Decide whether a failure should be retried. + * Retry only when: the error is transient, NO substantive content has been + * emitted yet (avoids dangerous duplicate regeneration), attempts remain, and + * the request was not intentionally aborted (user cancel / hard timeout). + */ +function shouldRetry( + err: unknown, + emittedContent: boolean, + attempt: number, + maxRetries: number, + aborted: boolean, +): boolean { + if (aborted) return false + if (emittedContent) return false + if (attempt >= maxRetries) return false + return isRetryableError(err) +} + +async function buildHttpError(response: Response, modelId: string): Promise { + const errorBody = await response.text().catch(() => "") + let message = `Command Code API error: ${response.status} ${response.statusText}` + let type = "" + try { + const parsed = JSON.parse(errorBody) + if (parsed?.error?.message) message = parsed.error.message + else if (parsed?.message) message = parsed.message + if (parsed?.error?.type) type = parsed.error.type + else if (parsed?.type) type = parsed.type + } catch { + // intentionally silent: error body is not JSON + } + const err = new Error(`${message} [model=${modelId}]`) + if (type) Object.assign(err, { code: type }) + return err +} + export interface CommandCodeModelOptions { apiKey: string baseURL?: string @@ -48,37 +244,207 @@ export class CommandCodeLanguageModel implements LanguageModelV3 { } } + /** + * Initial connection with retry/backoff for transient network and 5xx/429 + * failures. 4xx (auth, plan, credits, validation) are thrown immediately. + */ + private async fetchWithRetry(url: string, fetchOpts: RequestInit, maxRetries: number = MAX_RETRIES): Promise { + let lastError: unknown + for (let attempt = 0; attempt <= maxRetries; attempt++) { + try { + const response = await fetch(url, fetchOpts) + if (response.ok) return response + const err = await buildHttpError(response, this.modelId) + if (isRetryableStatus(response.status) && attempt < maxRetries) { + const delay = backoffDelay(attempt) + console.error( + `[CC-Retry] HTTP ${response.status} (attempt ${attempt + 1}/${maxRetries + 1}), retrying in ${Math.round(delay)}ms: ${err.message}`, + ) + await sleep(delay) + continue + } + throw err + } catch (err) { + lastError = err + if (!isRetryableError(err) || attempt >= maxRetries) throw err + const delay = backoffDelay(attempt) + console.error( + `[CC-Retry] network error (attempt ${attempt + 1}/${maxRetries + 1}), retrying in ${Math.round(delay)}ms: ${describeError(err)}`, + ) + await sleep(delay) + } + } + throw lastError + } + + /** + * Single fetch used for mid-stream reconnects. The retry budget for + * reconnects is owned by streamWithReconnect (no double counting). + * 5xx/429 are tagged with a `server_error` token so the wrapper classifies + * them as retryable; 4xx throw the parsed (non-retryable) error. + */ + private async fetchOnce(url: string, fetchOpts: RequestInit): Promise { + const response = await fetch(url, fetchOpts) + if (response.ok) return response + const err = await buildHttpError(response, this.modelId) + if (isRetryableStatus(response.status)) { + throw new Error(`server_error: reconnect HTTP ${response.status} (${err.message})`) + } + throw err + } + + /** + * Wraps a parsed stream to detect mid-stream disconnects and transparently + * reconnect — BUT only when no substantive content (text, reasoning, + * tool-call, tool-input) has been emitted yet. If partial output already + * went downstream, reconnecting would regenerate from scratch and produce + * DUPLICATE content, so we instead surface a clear error. + */ + private streamWithReconnect( + makeStream: () => ReadableStream, + reconnect: () => Promise>, + isAborted: () => boolean, + maxRetries: number = MAX_RETRIES, + ): ReadableStream { + let attempt = 0 + let currentStream = makeStream() + let reader = currentStream.getReader() + let emittedContent = false + let pendingReconnect = false + + return new ReadableStream({ + async pull(controller) { + while (true) { + if (pendingReconnect) { + pendingReconnect = false + try { + currentStream = await reconnect() + reader = currentStream.getReader() + continue + } catch (reconnectErr) { + if (shouldRetry(reconnectErr, emittedContent, attempt, maxRetries, isAborted())) { + attempt++ + const delay = backoffDelay(attempt - 1) + console.error( + `[CC-Retry-Stream] reconnect failed (attempt ${attempt}/${maxRetries}), retrying in ${Math.round(delay)}ms: ${describeError(reconnectErr)}`, + ) + await sleep(delay) + pendingReconnect = true + continue + } + if (emittedContent) { + controller.error(partialOutputError(reconnectErr)) + } else { + controller.error(wrapAsError(reconnectErr)) + } + return + } + } + + let readResult: ReadableStreamReadResult + try { + readResult = await reader.read() + } catch (err) { + if (shouldRetry(err, emittedContent, attempt, maxRetries, isAborted())) { + attempt++ + const delay = backoffDelay(attempt - 1) + console.error( + `[CC-Retry-Stream] mid-stream disconnect (attempt ${attempt}/${maxRetries}), reconnecting in ${Math.round(delay)}ms: ${describeError(err)}`, + ) + await sleep(delay) + pendingReconnect = true + continue + } + if (emittedContent) { + controller.error(partialOutputError(err)) + } else { + controller.error(wrapAsError(err)) + } + return + } + + const { done, value } = readResult + if (done) { + controller.close() + return + } + if (!value) continue + + // Track substantive content emission — gates safe reconnect. + if ( + value.type === "text-delta" || + value.type === "reasoning-delta" || + value.type === "tool-call" || + value.type === "tool-input-start" || + value.type === "tool-input-delta" + ) { + emittedContent = true + } + + // Defensive: stream.ts converts SSE errors into controller.error(), + // so an {type:"error"} part should never arrive here. If it does, + // treat it as a terminal failure with retry gating. + if (value.type === "error") { + const inner = (value as { error?: unknown }).error + if (shouldRetry(inner, emittedContent, attempt, maxRetries, isAborted())) { + attempt++ + const delay = backoffDelay(attempt - 1) + console.error( + `[CC-Retry-Stream] error part (attempt ${attempt}/${maxRetries}), reconnecting in ${Math.round(delay)}ms: ${describeError(inner)}`, + ) + await sleep(delay) + pendingReconnect = true + continue + } + if (emittedContent) { + controller.error(partialOutputError(inner)) + } else { + controller.error(wrapAsError(inner)) + } + return + } + + controller.enqueue(value) + return + } + }, + cancel() { + reader.cancel() + }, + }) + } + async doStream(options: LanguageModelV3CallOptions): Promise { const body = buildRequest(this.modelId, options) const requestBody = JSON.stringify(body) const controller = new AbortController() - const timeout = setTimeout(() => controller.abort(new Error("Request timed out after 5 minutes")), 300_000) + let userAborted = false + const timeout = setTimeout( + () => controller.abort(new Error("Request timed out after 5 minutes")), + REQUEST_TIMEOUT_MS, + ) const userSignal = options.abortSignal if (userSignal) { - const onAbort = () => controller.abort(userSignal.reason) + const onAbort = () => { + userAborted = true + controller.abort(userSignal.reason) + } userSignal.addEventListener("abort", onAbort, { once: true }) } - try { - const response = await fetch(`${this.baseURL}/alpha/generate`, { - method: "POST", - headers: this.buildHeaders(), - body: requestBody, - signal: controller.signal, - }) + const url = `${this.baseURL}/alpha/generate` + const fetchOpts = (): RequestInit => ({ + method: "POST", + headers: this.buildHeaders(), + body: requestBody, + signal: controller.signal, + }) - if (!response.ok) { - const errorBody = await response.text().catch(() => "") - let errorMessage = `Command Code API error: ${response.status} ${response.statusText}` - try { - const parsed = JSON.parse(errorBody) - if (parsed.error?.message) errorMessage = parsed.error.message - else if (parsed.message) errorMessage = parsed.message - } catch { - // intentionally silent: error body is not JSON - } throw new Error(`${errorMessage} [model=${this.modelId}]`) - } + const isAborted = () => userAborted || controller.signal.aborted + + try { + const response = await this.fetchWithRetry(url, fetchOpts()) if (!response.body) { throw new Error(`Command Code API returned no body [model=${this.modelId}]`) @@ -89,8 +455,15 @@ export class CommandCodeLanguageModel implements LanguageModelV3 { responseHeaders[k] = v }) + const makeStream = () => parseStreamEvents(response.body as ReadableStream) + const reconnect = async (): Promise> => { + const r = await this.fetchOnce(url, fetchOpts()) + if (!r.body) throw new Error("server_error: reconnect returned no body") + return parseStreamEvents(r.body as ReadableStream) + } + return { - stream: parseStreamEvents(response.body as ReadableStream), + stream: this.streamWithReconnect(makeStream, reconnect, isAborted), request: { body: requestBody }, response: { headers: responseHeaders }, } diff --git a/tests/helpers/mocks.ts b/tests/helpers/mocks.ts index 5fd4e46..099e14c 100644 --- a/tests/helpers/mocks.ts +++ b/tests/helpers/mocks.ts @@ -196,3 +196,71 @@ export function makeCallOptions(overrides: Partial = ...overrides, } } + +/** + * Mock fetch to return a sequence of responses (for retry testing). + * Each call to fetch returns the next response in the sequence. + * The last response is reused for any additional calls. + */ +export function mockFetchRetrySequence( + responses: Array<{ + ok?: boolean + status?: number + statusText?: string + body?: ReadableStream | null + headers?: Headers + errorBody?: string + }>, +): { calls: MockFetchCall[]; restore: () => void } { + const encoder = new TextEncoder() + const calls: MockFetchCall[] = [] + let callIndex = 0 + const original = globalThis.fetch + + globalThis.fetch = ((input: RequestInfo | URL, options?: RequestInit) => { + calls.push({ + url: typeof input === "string" ? input : input instanceof URL ? input.href : input.url, + options: options ?? {}, + }) + const idx = Math.min(callIndex, responses.length - 1) + callIndex++ + const resp = responses[idx] + const errorBody = resp.errorBody ?? "" + return Promise.resolve({ + ok: resp.ok ?? true, + status: resp.status ?? 200, + statusText: resp.statusText ?? "OK", + headers: resp.headers ?? new Headers(), + body: resp.body ?? null, + text: () => Promise.resolve(errorBody), + json: () => { + try { return Promise.resolve(JSON.parse(errorBody)) } catch { return Promise.resolve({}) } + }, + } as Response) + }) as typeof globalThis.fetch + + return { + calls, + restore: () => { + globalThis.fetch = original + }, + } +} + +/** + * Create a simple SSE stream from data strings. + */ +export function makeSSEStream(chunks: string[]): ReadableStream { + const encoder = new TextEncoder() + let i = 0 + return new ReadableStream({ + pull(controller) { + if (i >= chunks.length) { + controller.close() + return + } + controller.enqueue(encoder.encode(chunks[i])) + i++ + }, + }) +} diff --git a/tests/unit/model.test.ts b/tests/unit/model.test.ts index 0d7e188..0635e4b 100644 --- a/tests/unit/model.test.ts +++ b/tests/unit/model.test.ts @@ -1,6 +1,6 @@ import { expect, test, beforeAll, afterAll } from "bun:test" import { CommandCodeLanguageModel } from "../../src/model.js" -import { mockFetchTrack, mockFetchError, mockFetchStream, makeCallOptions } from "../helpers/mocks.js" +import { mockFetchTrack, mockFetchError, mockFetchStream, mockFetchRetrySequence, makeCallOptions } from "../helpers/mocks.js" const MODEL_ID = "test-model" const API_KEY = "sk-test-key" @@ -126,8 +126,14 @@ test("doStream throws descriptive error on non-OK response", async () => { restore() }) -test("doStream throws on HTTP error without JSON body", async () => { - const { restore } = mockFetchError(500, "Internal Server Error") +test("doStream throws on HTTP error without JSON body", { timeout: 30000 }, async () => { + // 500 is retryable — return it for all 4 attempts (1 initial + 3 retries) + const { restore } = mockFetchRetrySequence([ + { ok: false, status: 500, statusText: "Internal Server Error", errorBody: "" }, + { ok: false, status: 500, statusText: "Internal Server Error", errorBody: "" }, + { ok: false, status: 500, statusText: "Internal Server Error", errorBody: "" }, + { ok: false, status: 500, statusText: "Internal Server Error", errorBody: "" }, + ]) const model = makeModel() expect(model.doStream(makeCallOptions())).rejects.toThrow("Command Code API error: 500 Internal Server Error") restore() @@ -218,7 +224,7 @@ test("doGenerate handles tool calls", async () => { }) test("doStream includes model ID in error messages", async () => { - const { restore } = mockFetchError(500, "Server Error", JSON.stringify({ + const { restore } = mockFetchError(400, "Bad Request", JSON.stringify({ error: { message: "Something broke" }, })) const model = makeModel() @@ -231,3 +237,181 @@ test("doStream includes model ID in error messages", async () => { } restore() }) + +// --- Retry/Backoff tests --- + +test("fetchWithRetry retries on HTTP 500 then succeeds", { timeout: 15000 }, async () => { + const encoder = new TextEncoder() + const successBody = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode('data: {"type":"start"}\n\n')) + controller.close() + }, + }) + const { calls, restore } = mockFetchRetrySequence([ + { ok: false, status: 500, statusText: "Internal Server Error", errorBody: '{"error":{"message":"server oops"}}' }, + { ok: true, status: 200, body: successBody }, + ]) + const model = makeModel() + const result = await model.doStream(makeCallOptions()) + restore() + + // Should have made 2 fetch calls (1 failed + 1 success) + expect(calls).toHaveLength(2) + // Result should have a valid stream + expect(result.stream).toBeDefined() + const reader = result.stream.getReader() + const { done } = await reader.read() + expect(done).toBe(false) + reader.releaseLock() +}) + +test("fetchWithRetry retries on HTTP 429 then succeeds", { timeout: 15000 }, async () => { + const encoder = new TextEncoder() + const successBody = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode('data: {"type":"start"}\n\n')) + controller.close() + }, + }) + const { calls, restore } = mockFetchRetrySequence([ + { ok: false, status: 429, statusText: "Too Many Requests", errorBody: '{"error":{"message":"rate limited"}}' }, + { ok: true, status: 200, body: successBody }, + ]) + const model = makeModel() + const result = await model.doStream(makeCallOptions()) + restore() + + expect(calls).toHaveLength(2) + expect(result.stream).toBeDefined() +}) + +test("fetchWithRetry fails fast on 401 (non-retryable)", async () => { + const { calls, restore } = mockFetchRetrySequence([ + { ok: false, status: 401, statusText: "Unauthorized", errorBody: '{"error":{"message":"Invalid API key"}}' }, + ]) + const model = makeModel() + try { + await model.doStream(makeCallOptions()) + expect.unreachable("Should have thrown") + } catch (err) { + expect((err as Error).message).toContain("Invalid API key") + } + restore() + + // Should have made only 1 fetch call — no retries for 4xx + expect(calls).toHaveLength(1) +}) + +test("fetchWithRetry fails fast on 403 (non-retryable)", async () => { + const { calls, restore } = mockFetchRetrySequence([ + { ok: false, status: 403, statusText: "Forbidden", errorBody: '{"error":{"message":"Forbidden"}}' }, + ]) + const model = makeModel() + expect(model.doStream(makeCallOptions())).rejects.toThrow("Forbidden") + restore() + expect(calls).toHaveLength(1) +}) + +test("fetchWithRetry respects max retries and throws last error", { timeout: 30000 }, async () => { + const { calls, restore } = mockFetchRetrySequence([ + { ok: false, status: 500, statusText: "Server Error", errorBody: '{"error":{"message":"down"}}' }, + { ok: false, status: 500, statusText: "Server Error", errorBody: '{"error":{"message":"down"}}' }, + { ok: false, status: 500, statusText: "Server Error", errorBody: '{"error":{"message":"down"}}' }, + { ok: false, status: 500, statusText: "Server Error", errorBody: '{"error":{"message":"down"}}' }, + ]) + const model = makeModel() + try { + await model.doStream(makeCallOptions()) + expect.unreachable("Should have thrown") + } catch (err) { + expect((err as Error).message).toContain("down") + } + restore() + + // Should have made 4 fetch calls (1 initial + 3 retries) + expect(calls).toHaveLength(4) +}) + +test("fetchWithRetry fails fast on quota/auth error patterns in body", async () => { + const { calls, restore } = mockFetchRetrySequence([ + { ok: false, status: 400, statusText: "Bad Request", errorBody: '{"error":{"message":"insufficient credit balance"}}' }, + ]) + const model = makeModel() + try { + await model.doStream(makeCallOptions()) + expect.unreachable("Should have thrown") + } catch (err) { + expect((err as Error).message).toContain("insufficient credit") + } + restore() + expect(calls).toHaveLength(1) +}) + +test("fetchWithRetry fails fast on validation_error", async () => { + const { calls, restore } = mockFetchRetrySequence([ + { ok: false, status: 400, statusText: "Bad Request", errorBody: '{"type":"validation_error","message":"Invalid params"}' }, + ]) + const model = makeModel() + expect(model.doStream(makeCallOptions())).rejects.toThrow("Invalid params") + restore() + expect(calls).toHaveLength(1) +}) + +test("fetchWithRetry retries network errors (fetch throws)", { timeout: 15000 }, async () => { + const encoder = new TextEncoder() + const original = globalThis.fetch + let callCount = 0 + globalThis.fetch = ((async () => { + callCount++ + if (callCount === 1) { + throw new Error("fetch failed: ECONNRESET") + } + return { + ok: true, + status: 200, + statusText: "OK", + headers: new Headers(), + body: new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode('data: {"type":"start"}\n\n')) + controller.close() + }, + }), + text: () => Promise.resolve(""), + } as Response + }) as typeof globalThis.fetch) + + const model = makeModel() + const result = await model.doStream(makeCallOptions()) + globalThis.fetch = original + + expect(callCount).toBe(2) + expect(result.stream).toBeDefined() +}) + +test("doStream sends retry-specific log messages on 5xx", { timeout: 15000 }, async () => { + const encoder = new TextEncoder() + const successBody = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode('data: {"type":"start"}\n\n')) + controller.close() + }, + }) + const { restore } = mockFetchRetrySequence([ + { ok: false, status: 502, statusText: "Bad Gateway", errorBody: '{"error":{"message":"bad gateway"}}' }, + { ok: true, status: 200, body: successBody }, + ]) + + const errors: string[] = [] + const origError = console.error + console.error = (...args: any[]) => errors.push(args.join(" ")) + + const model = makeModel() + await model.doStream(makeCallOptions()) + + console.error = origError + restore() + + expect(errors.some((e) => e.includes("[CC-Retry]") && e.includes("HTTP 502"))).toBe(true) +})