From 651f344ce3004d8ddc226c49b33a15e2f5a86e4a Mon Sep 17 00:00:00 2001 From: TensorNull Date: Thu, 22 Oct 2026 15:45:43 +0800 Subject: [PATCH] ci: add bounded transient live-smoke retry --- .github/workflows/live-smoke.yml | 8 +- .github/workflows/publish.yml | 1 + RELEASING.md | 17 +- ROADMAP.md | 12 +- scripts/live-smoke.mjs | 194 +++++++++++++++++------ scripts/release-workflow-validation.mjs | 3 +- scripts/test-live-smoke-contract.mjs | 202 +++++++++++++++++++++++- tests/workflow-contract.test.mjs | 21 +++ 8 files changed, 390 insertions(+), 68 deletions(-) diff --git a/.github/workflows/live-smoke.yml b/.github/workflows/live-smoke.yml index fe24767..c7003f1 100644 --- a/.github/workflows/live-smoke.yml +++ b/.github/workflows/live-smoke.yml @@ -42,11 +42,13 @@ jobs: COMETAPI_KEY: ${{ secrets.COMETAPI_KEY }} COMETAPI_LIVE_SMOKE: "1" COMETAPI_SMOKE_MODEL: ${{ vars.COMETAPI_SMOKE_MODEL || 'gpt-5.6-sol' }} - COMETAPI_LIVE_REQUEST_LIMIT: "3" + COMETAPI_LIVE_RETRY_BUDGET: "1" + COMETAPI_LIVE_REQUEST_LIMIT: "4" COMETAPI_LIVE_MAX_OUTPUT_TOKENS: "16" COMETAPI_LIVE_REQUEST_TIMEOUT_MS: "60000" COMETAPI_LIVE_CONCURRENCY: "1" - # The test performs exactly three sequential requests (Models, Chat - # non-streaming, Responses streaming), stops on the first failure, and + # The test performs exactly three sequential logical requests (Models, + # Chat non-streaming, Responses streaming), permits one global official + # transient retry, stops on the first logical failure, and # never prints the credential. run: npm run test:live diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 90dd051..4e124f0 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -706,6 +706,7 @@ jobs: COMETAPI_KEY: ${{ secrets.COMETAPI_KEY }} COMETAPI_LIVE_SMOKE: "1" COMETAPI_SMOKE_MODEL: ${{ vars.COMETAPI_SMOKE_MODEL || 'gpt-5.6-sol' }} + COMETAPI_LIVE_RETRY_BUDGET: "0" COMETAPI_LIVE_REQUEST_LIMIT: "3" COMETAPI_LIVE_MAX_OUTPUT_TOKENS: "16" COMETAPI_LIVE_REQUEST_TIMEOUT_MS: "60000" diff --git a/RELEASING.md b/RELEASING.md index eb0b911..052ec4e 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -279,11 +279,13 @@ The repository maintains four independently auditable workflows: and 24, with Node.js 26 advisory. - `live-smoke.yml`: trusted scheduled and manual default-branch-only low-cost compatibility tests against the pinned CometAPI HTTPS endpoint. Each run makes - exactly three sequential requests, caps generated output at 16 tokens, applies - a 60-second timeout to each request, fixes concurrency at one, and stops on the - first failure. Standalone and release smoke jobs share one repository-wide - concurrency group. Scheduled and manual live execution requires - `LIVE_SMOKE_ENABLED=true`. + exactly three sequential logical requests, permits one global official + transient retry, caps total HTTP attempts at four, caps generated output at + 16 tokens, applies a 60-second timeout to each request, fixes concurrency at + one, and stops on the first logical failure. Standalone and release smoke jobs + share one repository-wide concurrency group. Scheduled and manual live + execution requires `LIVE_SMOKE_ENABLED=true`. The release-tag smoke sets the + retry budget to zero and retains a three-attempt limit. - `release-please.yml`: patch-only versioning, an explicit `cometapi` branch component, separate pull requests, and the pinned root-package title `chore(main): release `. It requires @@ -414,8 +416,9 @@ dist-tag. Before authorizing release: immutable GitHub prerelease. 9. Let the release workflow verify `immutable: true`, `main` ancestry, the exact tarball, and the trusted release-tag live smoke: exactly three - sequential requests, 16 output tokens, 60 seconds per request, concurrency - one, and stop on the first failure. + sequential logical requests, a zero retry budget, three HTTP attempts, 16 + output tokens, 60 seconds per request, concurrency one, and stop on the + first logical failure. 10. Approve publication of that same tarball through the protected `npm` environment with OIDC and provenance, or use the one-time alpha.1 bootstrap only when normal Trusted Publishing cannot create the new package. The diff --git a/ROADMAP.md b/ROADMAP.md index bbd43eb..00e5250 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -185,8 +185,9 @@ The readiness criteria remain: requires approval by the current release approver and permits self-review. - Default-branch CI is rerun successfully, the fail-closed content gate reports all violations together and then passes, and an explicitly authorized - protected live smoke passes within the three-request, 16-output-token, - 60-second-per-request, concurrency-one, stop-on-first-failure budget. + protected live smoke passes within the three-logical-request, one-global- + retry, four-HTTP-attempt, 16-output-token, 60-second-per-request, + concurrency-one, stop-on-first-logical-failure budget. ## 0.1.0-alpha.1: Registry Alpha @@ -432,8 +433,11 @@ The repository will maintain: - `ci.yml` for offline pull-request and default-branch checks. - `live-smoke.yml` for budgeted trusted compatibility tests from the default branch to the pinned CometAPI HTTPS endpoint. Each run is fixed at exactly - three sequential requests, a 16-token output cap, a 60-second per-request - timeout, concurrency one, and stop on the first failure. + three sequential logical requests, permits one global official transient + retry, caps total HTTP attempts at four, uses a 16-token output cap and a + 60-second per-request timeout, fixes concurrency at one, and stops on the + first logical failure. The release-tag smoke sets the retry budget to zero + and retains a three-attempt limit. - `release-please.yml` for the reviewed version and changelog PR and the corresponding immutable tag and GitHub Release. - `publish.yml` for trusted Release Please completion, immutable-release and diff --git a/scripts/live-smoke.mjs b/scripts/live-smoke.mjs index e67fd84..057bd2d 100644 --- a/scripts/live-smoke.mjs +++ b/scripts/live-smoke.mjs @@ -1,4 +1,9 @@ import assert from "node:assert/strict"; +import { + APIConnectionError, + APIConnectionTimeoutError, + APIError, +} from "openai"; import { CometAPI } from "../dist/index.js"; @@ -9,17 +14,19 @@ const ALLOWED_URLS = new Set([ `${LIVE_BASE_URL}/responses`, ]); -function readBudget(name, maximum) { +function readBudget(name, maximum, minimum = 1) { const value = Number(process.env[name] ?? maximum); - if (!Number.isInteger(value) || value < 1 || value > maximum) { + if (!Number.isInteger(value) || value < minimum || value > maximum) { throw new Error( - `${name} must be an integer between 1 and ${String(maximum)}.`, + `${name} must be an integer between ${String(minimum)} and ${String(maximum)}.`, ); } return value; } -const REQUEST_LIMIT = readBudget("COMETAPI_LIVE_REQUEST_LIMIT", 3); +const LOGICAL_REQUEST_COUNT = 3; +const RETRY_BUDGET = readBudget("COMETAPI_LIVE_RETRY_BUDGET", 1, 0); +const REQUEST_LIMIT = readBudget("COMETAPI_LIVE_REQUEST_LIMIT", 4); const OUTPUT_TOKEN_LIMIT = readBudget("COMETAPI_LIVE_MAX_OUTPUT_TOKENS", 16); const REQUEST_TIMEOUT_MS = readBudget( "COMETAPI_LIVE_REQUEST_TIMEOUT_MS", @@ -35,8 +42,13 @@ if ( throw new Error(`The live smoke endpoint is pinned to ${LIVE_BASE_URL}.`); } -if (REQUEST_LIMIT !== 3 || concurrency !== 1) { - throw new Error("The 0.1 live smoke requires exactly 3 sequential requests."); +if ( + REQUEST_LIMIT !== LOGICAL_REQUEST_COUNT + RETRY_BUDGET || + concurrency !== 1 +) { + throw new Error( + "The 0.1 live smoke requires 3 sequential requests and a request limit equal to the logical count plus the global retry budget.", + ); } if (process.env.COMETAPI_LIVE_SMOKE !== "1") { @@ -52,6 +64,89 @@ if (!process.env.COMETAPI_KEY) { } let requestCount = 0; +let retryBudget = RETRY_BUDGET; + +function classifyError(error) { + let category = "unknown"; + let status = null; + if (error instanceof APIConnectionTimeoutError) { + category = "timeout"; + } else if (error instanceof APIConnectionError) { + category = "connection"; + } + if ( + category === "unknown" && + error instanceof APIError && + Number.isInteger(error.status) + ) { + status = error.status; + category = "http"; + } + if (error instanceof globalThis.DOMException && error.name === "AbortError") { + category = "abort"; + } + if (error instanceof Error) { + const code = error.code ?? error.cause?.code; + if ( + typeof code === "string" && + [ + "ECONNABORTED", + "ECONNREFUSED", + "ECONNRESET", + "EAI_AGAIN", + "ENETUNREACH", + "ENOTFOUND", + "ETIMEDOUT", + ].includes(code) + ) { + category = "transport"; + } + } + const transient = + category === "timeout" || + category === "connection" || + category === "abort" || + category === "transport" || + (category === "http" && + (status === 408 || status === 429 || (status !== null && status >= 500))); + return { category, status, transient }; +} + +async function runLogicalRequest(label, operation) { + for (let attempt = 1; attempt <= 2; attempt += 1) { + const startedAt = Date.now(); + try { + const result = await operation(); + console.error( + JSON.stringify({ + attempt, + category: "success", + elapsed_ms: Date.now() - startedAt, + request: label, + status: null, + }), + ); + return result; + } catch (error) { + const classification = classifyError(error); + console.error( + JSON.stringify({ + attempt, + category: classification.category, + elapsed_ms: Date.now() - startedAt, + request: label, + status: classification.status, + }), + ); + if (retryBudget === 0 || !classification.transient) { + throw error; + } + retryBudget -= 1; + } + } + throw new Error("The live smoke retry loop exhausted unexpectedly."); +} + const guardedFetch = async (input, init) => { requestCount += 1; if (requestCount > REQUEST_LIMIT) { @@ -82,15 +177,17 @@ const client = new CometAPI({ timeout: REQUEST_TIMEOUT_MS, }); -const models = await client.models.list(); +const models = await runLogicalRequest("Models", () => client.models.list()); assert.ok(models.data.length > 0, "models.list returned no models"); -const chat = await client.chat.completions.create({ - max_completion_tokens: OUTPUT_TOKEN_LIMIT, - messages: [{ content: "Reply with OK.", role: "user" }], - model, - reasoning_effort: "none", -}); +const chat = await runLogicalRequest("Chat Completions", () => + client.chat.completions.create({ + max_completion_tokens: OUTPUT_TOKEN_LIMIT, + messages: [{ content: "Reply with OK.", role: "user" }], + model, + reasoning_effort: "none", + }), +); assert.ok(chat.id, "Chat Completions returned no response ID"); const chatChoice = chat.choices[0]; assert.ok(chatChoice, "Chat Completions returned no choices"); @@ -104,41 +201,48 @@ assert.ok( "Chat Completions returned no assistant text", ); -const stream = await client.responses.create({ - input: "Reply with OK.", - max_output_tokens: OUTPUT_TOKEN_LIMIT, - model, - reasoning: { effort: "none" }, - stream: true, -}); -let eventCount = 0; -let responseCompleted = false; -let responseText = ""; -for await (const event of stream) { - eventCount += 1; - if ( - event.type === "error" || - event.type === "response.failed" || - event.type === "response.incomplete" - ) { - throw new Error(`Responses stream ended with ${event.type}.`); +const response = await runLogicalRequest("Responses", async () => { + const stream = await client.responses.create({ + input: "Reply with OK.", + max_output_tokens: OUTPUT_TOKEN_LIMIT, + model, + reasoning: { effort: "none" }, + stream: true, + }); + let eventCount = 0; + let responseCompleted = false; + let responseText = ""; + for await (const event of stream) { + eventCount += 1; + if ( + event.type === "error" || + event.type === "response.failed" || + event.type === "response.incomplete" + ) { + throw new Error(`Responses stream ended with ${event.type}.`); + } + if (event.type === "response.output_text.delta") { + responseText += event.delta; + } + if (event.type === "response.completed") { + assert.equal( + event.response.status, + "completed", + "Responses terminal event was not completed", + ); + responseCompleted = true; + } } - if (event.type === "response.output_text.delta") { - responseText += event.delta; - } - if (event.type === "response.completed") { - assert.equal( - event.response.status, - "completed", - "Responses terminal event was not completed", - ); - responseCompleted = true; - } -} + return { eventCount, responseCompleted, responseText }; +}); +const { eventCount, responseCompleted, responseText } = response; assert.ok(eventCount > 0, "Responses stream returned no events"); assert.ok(responseCompleted, "Responses stream returned no completed event"); assert.ok(responseText.trim(), "Responses stream returned no output text"); -assert.equal(requestCount, REQUEST_LIMIT); +assert.ok( + requestCount >= LOGICAL_REQUEST_COUNT && requestCount <= REQUEST_LIMIT, + `Live smoke used ${String(requestCount)} HTTP attempts for ${String(LOGICAL_REQUEST_COUNT)} logical requests.`, +); console.log( - `Live smoke passed ${String(requestCount)} sequential requests with a ${String(OUTPUT_TOKEN_LIMIT)}-token output cap.`, + `Live smoke passed ${String(LOGICAL_REQUEST_COUNT)} sequential requests in ${String(requestCount)} HTTP attempts with a ${String(OUTPUT_TOKEN_LIMIT)}-token output cap.`, ); diff --git a/scripts/release-workflow-validation.mjs b/scripts/release-workflow-validation.mjs index 826a47f..ef00b5b 100644 --- a/scripts/release-workflow-validation.mjs +++ b/scripts/release-workflow-validation.mjs @@ -18,7 +18,7 @@ const RELEASE_WORKFLOW_STEP = "Run Release Please"; const PUBLISH_OPERATION = "release"; const NPM_TAG_POLICY_ID = 55718965; const PUBLISH_WORKFLOW_CONTRACT_SHA256 = - "081af3a47c0bef1ff38a77cf038ab3c66b8c14ea2b524c42bc4f2c21e41f9f79"; + "684e15c13835ae3b139929ae78e5bbd5ba618ea8f30ebba8dbb4c42004d57a06"; const PUBLISH_HANDOFF_IF = "vars.RELEASE_PLEASE_ENABLED == 'true' && github.event_name == 'workflow_run' && github.run_attempt == 1 && github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push' && github.event.workflow_run.head_branch == 'main'"; const PUBLISH_RESULT_IF = "steps.result.outputs.has-result == 'true'"; @@ -570,6 +570,7 @@ fi COMETAPI_KEY: "${{ secrets.COMETAPI_KEY }}", COMETAPI_LIVE_SMOKE: "1", COMETAPI_SMOKE_MODEL: "${{ vars.COMETAPI_SMOKE_MODEL || 'gpt-5.6-sol' }}", + COMETAPI_LIVE_RETRY_BUDGET: "0", COMETAPI_LIVE_REQUEST_LIMIT: "3", COMETAPI_LIVE_MAX_OUTPUT_TOKENS: "16", COMETAPI_LIVE_REQUEST_TIMEOUT_MS: "60000", diff --git a/scripts/test-live-smoke-contract.mjs b/scripts/test-live-smoke-contract.mjs index 012ff03..d992c14 100644 --- a/scripts/test-live-smoke-contract.mjs +++ b/scripts/test-live-smoke-contract.mjs @@ -1,5 +1,7 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; +import { TextEncoder } from "node:util"; +import { ReadableStream } from "node:stream/web"; const script = new globalThis.URL("./live-smoke.mjs", import.meta.url); const LIVE_BASE_URL = "https://api.cometapi.com/v1"; @@ -15,6 +17,7 @@ const environmentNames = [ "COMETAPI_BASE_URL", "COMETAPI_KEY", "COMETAPI_LIVE_CONCURRENCY", + "COMETAPI_LIVE_RETRY_BUDGET", "COMETAPI_LIVE_MAX_OUTPUT_TOKENS", "COMETAPI_LIVE_REQUEST_LIMIT", "COMETAPI_LIVE_REQUEST_TIMEOUT_MS", @@ -25,9 +28,10 @@ const originalEnvironment = new Map( environmentNames.map((name) => [name, process.env[name]]), ); -function json(body) { +function json(body, { headers = {}, status = 200 } = {}) { return new Response(JSON.stringify(body), { - headers: { "content-type": "application/json" }, + headers: { "content-type": "application/json", ...headers }, + status, }); } @@ -42,6 +46,33 @@ function sse(messages) { }); } +function interruptedSse() { + const encoder = new TextEncoder(); + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + encoder.encode( + `event: response.output_text.delta\ndata: ${JSON.stringify({ + content_index: 0, + delta: "partial", + item_id: "msg_live_contract", + output_index: 0, + sequence_number: 0, + type: "response.output_text.delta", + })}\n\n`, + ), + ); + const error = Object.assign(new Error("stream interrupted"), { + code: "ECONNRESET", + }); + controller.error(error); + }, + }), + { headers: { "content-type": "text/event-stream" } }, + ); +} + function completedResponse() { return { created_at: 1, @@ -60,7 +91,12 @@ function createMockFetch(scenario, requestURLs = []) { return async (input, init) => { const request = new Request(input, init); requestURLs.push(request.url); - assert.equal(request.headers.get("authorization"), "Bearer mock-live-key"); + if (scenario !== "redaction") { + assert.equal( + request.headers.get("authorization"), + "Bearer mock-live-key", + ); + } if (request.url.endsWith("/models")) { return json({ @@ -82,7 +118,7 @@ function createMockFetch(scenario, requestURLs = []) { assert.equal(body.reasoning_effort, "none"); return json({ choices: - scenario === "empty-chat" + scenario === "empty-chat" || scenario === "redaction" ? [] : [ { @@ -99,6 +135,70 @@ function createMockFetch(scenario, requestURLs = []) { } assert.ok(request.url.endsWith("/responses")); + if ( + scenario === "unauthorized-response" && + requestURLs.filter((url) => url.endsWith("/responses")).length === 1 + ) { + return json( + { + error: { + code: "invalid_api_key", + message: "unauthorized", + type: "authentication_error", + }, + }, + { status: 401 }, + ); + } + if ( + (scenario === "timeout-response" || scenario === "retry-exhausted") && + (scenario === "retry-exhausted" || + requestURLs.filter((url) => url.endsWith("/responses")).length === 1) + ) { + throw new globalThis.DOMException( + "The operation was aborted", + "AbortError", + ); + } + if ( + scenario === "connection-response" && + requestURLs.filter((url) => url.endsWith("/responses")).length === 1 + ) { + throw new TypeError("fetch failed"); + } + if ( + scenario === "stream-interruption" && + requestURLs.filter((url) => url.endsWith("/responses")).length === 1 + ) { + return interruptedSse(); + } + if ( + scenario === "retry-response" && + requestURLs.filter((url) => url.endsWith("/responses")).length === 1 + ) { + return json( + { + error: { + code: "server_error", + message: "temporary response failure", + type: "server_error", + }, + }, + { headers: { "retry-after-ms": "0" }, status: 500 }, + ); + } + if (scenario === "auth-error") { + return json( + { + error: { + code: "invalid_api_key", + message: "mock authentication failure", + type: "authentication_error", + }, + }, + { status: 401 }, + ); + } const body = await request.clone().json(); assert.equal(body.model, DEFAULT_MODEL); assert.deepEqual(body.reasoning, { effort: "none" }); @@ -184,13 +284,25 @@ async function runSuccessfulScenario() { assert.deepEqual(requestURLs, FULL_REQUEST_SEQUENCE); } -function runFailingScenario(scenario, environment = {}) { +async function runRetryScenario() { + const requestURLs = []; + globalThis.fetch = createMockFetch("retry-response", requestURLs); + await import(`${script.href}?scenario=retry-response`); + assert.deepEqual(requestURLs, [ + ...FULL_REQUEST_SEQUENCE, + `${LIVE_BASE_URL}/responses`, + ]); +} + +function runScenario(scenario, environment = {}) { const childProgram = [ 'import assert from "node:assert/strict";', `const DEFAULT_MODEL = ${JSON.stringify(DEFAULT_MODEL)};`, `const requestURLs = [];`, + "console.log = () => undefined;", json.toString(), sse.toString(), + interruptedSse.toString(), completedResponse.toString(), createMockFetch.toString(), `globalThis.fetch = createMockFetch(${JSON.stringify(scenario)}, requestURLs);`, @@ -204,8 +316,9 @@ function runFailingScenario(scenario, environment = {}) { ...process.env, COMETAPI_KEY: "mock-live-key", COMETAPI_LIVE_CONCURRENCY: "1", + COMETAPI_LIVE_RETRY_BUDGET: "1", COMETAPI_LIVE_MAX_OUTPUT_TOKENS: "16", - COMETAPI_LIVE_REQUEST_LIMIT: "3", + COMETAPI_LIVE_REQUEST_LIMIT: "4", COMETAPI_LIVE_REQUEST_TIMEOUT_MS: "60000", COMETAPI_LIVE_SMOKE: "1", ...environment, @@ -230,17 +343,24 @@ function assertFailingScenario({ expectedRequests, scenario, }) { - const result = runFailingScenario(scenario, environment); + const result = runScenario(scenario, environment); assert.notEqual(result.status, 0, `${scenario} must fail the live runner`); assert.match(result.stderr, expectedError); assert.deepEqual(JSON.parse(result.stdout), expectedRequests); } +function assertSuccessfulScenario({ environment, expectedRequests, scenario }) { + const result = runScenario(scenario, environment); + assert.equal(result.status, 0, `${scenario} must pass the live runner`); + assert.deepEqual(JSON.parse(result.stdout), expectedRequests); +} + try { console.log = () => undefined; process.env.COMETAPI_KEY = "mock-live-key"; process.env.COMETAPI_LIVE_SMOKE = "1"; - process.env.COMETAPI_LIVE_REQUEST_LIMIT = "3"; + process.env.COMETAPI_LIVE_RETRY_BUDGET = "1"; + process.env.COMETAPI_LIVE_REQUEST_LIMIT = "4"; process.env.COMETAPI_LIVE_MAX_OUTPUT_TOKENS = "16"; process.env.COMETAPI_LIVE_REQUEST_TIMEOUT_MS = "60000"; process.env.COMETAPI_LIVE_CONCURRENCY = "1"; @@ -248,6 +368,72 @@ try { delete process.env.COMETAPI_SMOKE_MODEL; await runSuccessfulScenario(); + await runRetryScenario(); + assertSuccessfulScenario({ + expectedRequests: [...FULL_REQUEST_SEQUENCE, `${LIVE_BASE_URL}/responses`], + scenario: "retry-response", + }); + for (const scenario of [ + "connection-response", + "timeout-response", + "stream-interruption", + ]) { + assertSuccessfulScenario({ + expectedRequests: [ + ...FULL_REQUEST_SEQUENCE, + `${LIVE_BASE_URL}/responses`, + ], + scenario, + }); + } + assertFailingScenario({ + environment: { COMETAPI_LIVE_RETRY_BUDGET: "2" }, + expectedError: /between 0 and 1/, + expectedRequests: [], + scenario: "invalid-retry-budget", + }); + assertFailingScenario({ + environment: { + COMETAPI_LIVE_RETRY_BUDGET: "1", + COMETAPI_LIVE_REQUEST_LIMIT: "3", + }, + expectedError: /logical count plus the global retry budget/, + expectedRequests: [], + scenario: "invalid-request-budget", + }); + assertFailingScenario({ + environment: { + COMETAPI_LIVE_RETRY_BUDGET: "0", + COMETAPI_LIVE_REQUEST_LIMIT: "3", + }, + expectedError: /operation was aborted|Request timed out/, + expectedRequests: FULL_REQUEST_SEQUENCE, + scenario: "timeout-response", + }); + assertFailingScenario({ + expectedError: /operation was aborted|Request timed out/, + expectedRequests: [...FULL_REQUEST_SEQUENCE, `${LIVE_BASE_URL}/responses`], + scenario: "retry-exhausted", + }); + assertFailingScenario({ + expectedError: /401/, + expectedRequests: FULL_REQUEST_SEQUENCE, + scenario: "auth-error", + }); + assertFailingScenario({ + expectedError: /401|unauthorized|authentication/i, + expectedRequests: FULL_REQUEST_SEQUENCE, + scenario: "unauthorized-response", + }); + const redactionSecret = "mock-live-secret-must-not-leak"; + const redaction = runScenario("redaction", { + COMETAPI_KEY: redactionSecret, + }); + assert.notEqual(redaction.status, 0); + assert.doesNotMatch( + `${redaction.stdout}\n${redaction.stderr}`, + new RegExp(redactionSecret), + ); assertFailingScenario({ environment: { COMETAPI_BASE_URL: "https://attacker.invalid/v1" }, expectedError: /endpoint is pinned/, diff --git a/tests/workflow-contract.test.mjs b/tests/workflow-contract.test.mjs index dbf4465..dfcf642 100644 --- a/tests/workflow-contract.test.mjs +++ b/tests/workflow-contract.test.mjs @@ -79,6 +79,27 @@ describe("GitHub Actions workflow contract", () => { ); }); + it("keeps transient live retries bounded in scheduled and release smoke", () => { + for (const name of ["live-smoke.yml", "publish.yml"]) { + const smoke = job( + workflow(name), + name === "live-smoke.yml" ? "smoke" : "live-smoke", + ); + expect(smoke).toContain( + name === "live-smoke.yml" + ? 'COMETAPI_LIVE_RETRY_BUDGET: "1"' + : 'COMETAPI_LIVE_RETRY_BUDGET: "0"', + ); + expect(smoke).toContain( + name === "live-smoke.yml" + ? 'COMETAPI_LIVE_REQUEST_LIMIT: "4"' + : 'COMETAPI_LIVE_REQUEST_LIMIT: "3"', + ); + expect(smoke).toContain('COMETAPI_LIVE_REQUEST_TIMEOUT_MS: "60000"'); + expect(smoke).toContain('COMETAPI_LIVE_CONCURRENCY: "1"'); + } + }); + it("keeps supported runtimes blocking and Node.js 26 advisory", () => { const ci = workflow("ci.yml"); const locked = job(ci, "locked");