11import assert from "node:assert/strict" ;
2+ import {
3+ APIConnectionError ,
4+ APIConnectionTimeoutError ,
5+ APIError ,
6+ } from "openai" ;
27
38import { CometAPI } from "../dist/index.js" ;
49
@@ -9,17 +14,19 @@ const ALLOWED_URLS = new Set([
914 `${ LIVE_BASE_URL } ,
1015] ) ;
1116
12- function readBudget ( name , maximum ) {
17+ function readBudget ( name , maximum , minimum = 1 ) {
1318 const value = Number ( process . env [ name ] ?? maximum ) ;
14- if ( ! Number . isInteger ( value ) || value < 1 || value > maximum ) {
19+ if ( ! Number . isInteger ( value ) || value < minimum || value > maximum ) {
1520 throw new Error (
16- `${ name } 1 and ${ String ( maximum ) } ,
21+ `${ name } ${ String ( minimum ) } ${ String ( maximum ) } ,
1722 ) ;
1823 }
1924 return value ;
2025}
2126
22- const REQUEST_LIMIT = readBudget ( "COMETAPI_LIVE_REQUEST_LIMIT" , 3 ) ;
27+ const LOGICAL_REQUEST_COUNT = 3 ;
28+ const RETRY_BUDGET = readBudget ( "COMETAPI_LIVE_RETRY_BUDGET" , 1 , 0 ) ;
29+ const REQUEST_LIMIT = readBudget ( "COMETAPI_LIVE_REQUEST_LIMIT" , 4 ) ;
2330const OUTPUT_TOKEN_LIMIT = readBudget ( "COMETAPI_LIVE_MAX_OUTPUT_TOKENS" , 16 ) ;
2431const REQUEST_TIMEOUT_MS = readBudget (
2532 "COMETAPI_LIVE_REQUEST_TIMEOUT_MS" ,
3542 throw new Error ( `The live smoke endpoint is pinned to ${ LIVE_BASE_URL } ) ;
3643}
3744
38- if ( REQUEST_LIMIT !== 3 || concurrency !== 1 ) {
39- throw new Error ( "The 0.1 live smoke requires exactly 3 sequential requests." ) ;
45+ if (
46+ REQUEST_LIMIT !== LOGICAL_REQUEST_COUNT + RETRY_BUDGET ||
47+ concurrency !== 1
48+ ) {
49+ throw new Error (
50+ "The 0.1 live smoke requires 3 sequential requests and a request limit equal to the logical count plus the global retry budget." ,
51+ ) ;
4052}
4153
4254if ( process . env . COMETAPI_LIVE_SMOKE !== "1" ) {
@@ -52,6 +64,89 @@ if (!process.env.COMETAPI_KEY) {
5264}
5365
5466let requestCount = 0 ;
67+ let retryBudget = RETRY_BUDGET ;
68+
69+ function classifyError ( error ) {
70+ let category = "unknown" ;
71+ let status = null ;
72+ if ( error instanceof APIConnectionTimeoutError ) {
73+ category = "timeout" ;
74+ } else if ( error instanceof APIConnectionError ) {
75+ category = "connection" ;
76+ }
77+ if (
78+ category === "unknown" &&
79+ error instanceof APIError &&
80+ Number . isInteger ( error . status )
81+ ) {
82+ status = error . status ;
83+ category = "http" ;
84+ }
85+ if ( error instanceof globalThis . DOMException && error . name === "AbortError" ) {
86+ category = "abort" ;
87+ }
88+ if ( error instanceof Error ) {
89+ const code = error . code ?? error . cause ?. code ;
90+ if (
91+ typeof code === "string" &&
92+ [
93+ "ECONNABORTED" ,
94+ "ECONNREFUSED" ,
95+ "ECONNRESET" ,
96+ "EAI_AGAIN" ,
97+ "ENETUNREACH" ,
98+ "ENOTFOUND" ,
99+ "ETIMEDOUT" ,
100+ ] . includes ( code )
101+ ) {
102+ category = "transport" ;
103+ }
104+ }
105+ const transient =
106+ category === "timeout" ||
107+ category === "connection" ||
108+ category === "abort" ||
109+ category === "transport" ||
110+ ( category === "http" &&
111+ ( status === 408 || status === 429 || ( status !== null && status >= 500 ) ) ) ;
112+ return { category, status, transient } ;
113+ }
114+
115+ async function runLogicalRequest ( label , operation ) {
116+ for ( let attempt = 1 ; attempt <= 2 ; attempt += 1 ) {
117+ const startedAt = Date . now ( ) ;
118+ try {
119+ const result = await operation ( ) ;
120+ console . error (
121+ JSON . stringify ( {
122+ attempt,
123+ category : "success" ,
124+ elapsed_ms : Date . now ( ) - startedAt ,
125+ request : label ,
126+ status : null ,
127+ } ) ,
128+ ) ;
129+ return result ;
130+ } catch ( error ) {
131+ const classification = classifyError ( error ) ;
132+ console . error (
133+ JSON . stringify ( {
134+ attempt,
135+ category : classification . category ,
136+ elapsed_ms : Date . now ( ) - startedAt ,
137+ request : label ,
138+ status : classification . status ,
139+ } ) ,
140+ ) ;
141+ if ( retryBudget === 0 || ! classification . transient ) {
142+ throw error ;
143+ }
144+ retryBudget -= 1 ;
145+ }
146+ }
147+ throw new Error ( "The live smoke retry loop exhausted unexpectedly." ) ;
148+ }
149+
55150const guardedFetch = async ( input , init ) => {
56151 requestCount += 1 ;
57152 if ( requestCount > REQUEST_LIMIT ) {
@@ -82,15 +177,17 @@ const client = new CometAPI({
82177 timeout : REQUEST_TIMEOUT_MS ,
83178} ) ;
84179
85- const models = await client . models . list ( ) ;
180+ const models = await runLogicalRequest ( "Models" , ( ) => client . models . list ( ) ) ;
86181assert . ok ( models . data . length > 0 , "models.list returned no models" ) ;
87182
88- const chat = await client . chat . completions . create ( {
89- max_completion_tokens : OUTPUT_TOKEN_LIMIT ,
90- messages : [ { content : "Reply with OK." , role : "user" } ] ,
91- model,
92- reasoning_effort : "none" ,
93- } ) ;
183+ const chat = await runLogicalRequest ( "Chat Completions" , ( ) =>
184+ client . chat . completions . create ( {
185+ max_completion_tokens : OUTPUT_TOKEN_LIMIT ,
186+ messages : [ { content : "Reply with OK." , role : "user" } ] ,
187+ model,
188+ reasoning_effort : "none" ,
189+ } ) ,
190+ ) ;
94191assert . ok ( chat . id , "Chat Completions returned no response ID" ) ;
95192const chatChoice = chat . choices [ 0 ] ;
96193assert . ok ( chatChoice , "Chat Completions returned no choices" ) ;
@@ -104,41 +201,48 @@ assert.ok(
104201 "Chat Completions returned no assistant text" ,
105202) ;
106203
107- const stream = await client . responses . create ( {
108- input : "Reply with OK." ,
109- max_output_tokens : OUTPUT_TOKEN_LIMIT ,
110- model,
111- reasoning : { effort : "none" } ,
112- stream : true ,
113- } ) ;
114- let eventCount = 0 ;
115- let responseCompleted = false ;
116- let responseText = "" ;
117- for await ( const event of stream ) {
118- eventCount += 1 ;
119- if (
120- event . type === "error" ||
121- event . type === "response.failed" ||
122- event . type === "response.incomplete"
123- ) {
124- throw new Error ( `Responses stream ended with ${ event . type } ) ;
204+ const response = await runLogicalRequest ( "Responses" , async ( ) => {
205+ const stream = await client . responses . create ( {
206+ input : "Reply with OK." ,
207+ max_output_tokens : OUTPUT_TOKEN_LIMIT ,
208+ model,
209+ reasoning : { effort : "none" } ,
210+ stream : true ,
211+ } ) ;
212+ let eventCount = 0 ;
213+ let responseCompleted = false ;
214+ let responseText = "" ;
215+ for await ( const event of stream ) {
216+ eventCount += 1 ;
217+ if (
218+ event . type === "error" ||
219+ event . type === "response.failed" ||
220+ event . type === "response.incomplete"
221+ ) {
222+ throw new Error ( `Responses stream ended with ${ event . type } ) ;
223+ }
224+ if ( event . type === "response.output_text.delta" ) {
225+ responseText += event . delta ;
226+ }
227+ if ( event . type === "response.completed" ) {
228+ assert . equal (
229+ event . response . status ,
230+ "completed" ,
231+ "Responses terminal event was not completed" ,
232+ ) ;
233+ responseCompleted = true ;
234+ }
125235 }
126- if ( event . type === "response.output_text.delta" ) {
127- responseText += event . delta ;
128- }
129- if ( event . type === "response.completed" ) {
130- assert . equal (
131- event . response . status ,
132- "completed" ,
133- "Responses terminal event was not completed" ,
134- ) ;
135- responseCompleted = true ;
136- }
137- }
236+ return { eventCount, responseCompleted, responseText } ;
237+ } ) ;
238+ const { eventCount, responseCompleted, responseText } = response ;
138239assert . ok ( eventCount > 0 , "Responses stream returned no events" ) ;
139240assert . ok ( responseCompleted , "Responses stream returned no completed event" ) ;
140241assert . ok ( responseText . trim ( ) , "Responses stream returned no output text" ) ;
141- assert . equal ( requestCount , REQUEST_LIMIT ) ;
242+ assert . ok (
243+ requestCount >= LOGICAL_REQUEST_COUNT && requestCount <= REQUEST_LIMIT ,
244+ `Live smoke used ${ String ( requestCount ) } ${ String ( LOGICAL_REQUEST_COUNT ) } ,
245+ ) ;
142246console . log (
143- `Live smoke passed ${ String ( requestCount ) } ${ String ( OUTPUT_TOKEN_LIMIT ) } ,
247+ `Live smoke passed ${ String ( LOGICAL_REQUEST_COUNT ) } in ${ String ( requestCount ) } HTTP attempts with a ${ String ( OUTPUT_TOKEN_LIMIT ) } ,
144248) ;
0 commit comments