Skip to content

Commit 7762c62

Browse files
committed
refactor(openai_media): 重构图片生成与编辑逻辑,优化认证与错误处理
1. 新增SSO令牌传递与多账户重试机制,修复账户资源耗尽问题 2. 将图片base64转换改为通过认证传输层获取,适配需要鉴权的Grok图片资源 3. 重构captureImageURLs为流式调用并添加错误返回,提升调用可靠性 4. 统一错误处理流程,返回更明确的服务端错误信息 5. 优化批量图片生成的错误捕获与返回逻辑
1 parent 21bb722 commit 7762c62

1 file changed

Lines changed: 159 additions & 41 deletions

File tree

internal/api/openai_media.go

Lines changed: 159 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -122,14 +122,22 @@ func (s *Server) handleImageGenerations(c *gin.Context) {
122122
n = maxN
123123
}
124124

125+
apiToken, _ := c.Get("api_token")
126+
ssoToken, _ := apiToken.(string)
127+
125128
prompt := "Drawing: " + req.Prompt
126-
imageURLs := s.captureLiteImageBatch(c.Request, spec, prompt, n)
129+
imageURLs, genErr := s.captureLiteImageBatch(c.Request, spec, prompt, n, ssoToken)
130+
131+
if len(imageURLs) == 0 && genErr != nil {
132+
writeAppError(c, genErr)
133+
return
134+
}
127135

128136
out := []map[string]any{}
129137
for i := 0; i < n && i < len(imageURLs); i++ {
130138
url := imageURLs[i]
131139
if responseFormat == "b64_json" {
132-
b64, err := fetchImageBase64(url)
140+
b64, err := s.fetchImageBase64ViaTransport(url)
133141
if err == nil {
134142
out = append(out, map[string]any{"b64_json": b64})
135143
continue
@@ -200,7 +208,7 @@ func (s *Server) handleWSImageGenerations(c *gin.Context, spec *model.Spec, prom
200208
if img.blob != "" {
201209
out = append(out, map[string]any{"b64_json": img.blob})
202210
} else if img.url != "" {
203-
b64, err := fetchImageBase64(img.url)
211+
b64, err := s.fetchImageBase64ViaTransport(img.url)
204212
if err == nil {
205213
out = append(out, map[string]any{"b64_json": b64})
206214
continue
@@ -266,15 +274,18 @@ func (s *Server) handleImageEdits(c *gin.Context) {
266274
"image_url": map[string]any{"url": dataURI},
267275
})
268276
}
277+
apiToken, _ := c.Get("api_token")
278+
ssoToken, _ := apiToken.(string)
279+
269280
messages := []map[string]any{{"role": "user", "content": contentBlocks}}
270281
chatReq := &chatCompletionRequest{Model: modelName, Messages: messages}
271282
streamOff := false
272283
chatReq.Stream = &streamOff
273-
imageURLs := s.captureImageURLs(c.Request, chatReq, spec)
284+
imageURLs, _ := s.captureImageURLs(c.Request, chatReq, spec, ssoToken)
274285
out := []map[string]any{}
275286
for _, url := range imageURLs {
276287
if responseFormat == "b64_json" {
277-
b64, err := fetchImageBase64(url)
288+
b64, err := s.fetchImageBase64ViaTransport(url)
278289
if err == nil {
279290
out = append(out, map[string]any{"b64_json": b64})
280291
continue
@@ -285,59 +296,138 @@ func (s *Server) handleImageEdits(c *gin.Context) {
285296
c.JSON(http.StatusOK, gin.H{"created": time.Now().Unix(), "data": out})
286297
}
287298

288-
// captureImageURLs runs the non-streaming chat path and extracts any image URLs.
289-
func (s *Server) captureImageURLs(r *http.Request, req *chatCompletionRequest, spec *model.Spec) []string {
290-
cw := &captureWriter{}
291-
292-
lease, _ := reserveAccount(r.Context(), s.Directory, spec, nil)
293-
if lease == nil {
294-
return nil
295-
}
296-
defer s.Directory.Release(lease)
297-
298-
emitThink := resolveEmitThink(req.ReasoningEffort)
299+
// captureImageURLs runs the STREAMING chat path (same as /v1/chat/completions)
300+
// and extracts any image URLs from the response. Using the streaming path is
301+
// more reliable because grok's non-streaming responses sometimes omit the
302+
// image URL even when progress reaches 100%.
303+
// ssoToken is the Bearer token from the request, used as fallback when the pool is empty.
304+
func (s *Server) captureImageURLs(r *http.Request, req *chatCompletionRequest, spec *model.Spec, ssoToken string) ([]string, error) {
299305
message, fileInputs, perr := extractMessages(req.Messages)
300306
if perr != nil {
301-
return nil
307+
return nil, perr
302308
}
303-
temp := 0.8
304-
if req.Temperature != nil {
305-
temp = *req.Temperature
309+
310+
maxRetries := selectionMaxRetries()
311+
exclude := []string{}
312+
var lastErr error
313+
314+
for attempt := 0; attempt <= maxRetries; attempt++ {
315+
lease, _ := reserveAccount(r.Context(), s.Directory, spec, exclude)
316+
if lease == nil {
317+
if s.Refresh != nil {
318+
_ = s.Refresh.RefreshOnDemand(r.Context())
319+
lease, _ = reserveAccount(r.Context(), s.Directory, spec, exclude)
320+
}
321+
}
322+
if lease == nil && ssoToken != "" {
323+
lease = &account.Lease{Token: ssoToken, ModeID: int(spec.ModeId)}
324+
}
325+
if lease == nil {
326+
return nil, platform.RateLimitError("No available accounts")
327+
}
328+
329+
urls, err := s.captureImageURLsOnce(r, lease, spec, message, fileInputs)
330+
s.Directory.Release(lease)
331+
332+
if err != nil {
333+
lastErr = err
334+
if attempt < maxRetries {
335+
exclude = append(exclude, lease.Token)
336+
}
337+
continue
338+
}
339+
if len(urls) > 0 {
340+
return urls, nil
341+
}
342+
343+
// Got a valid response but no image URLs — retry with a different account.
344+
if attempt < maxRetries {
345+
exclude = append(exclude, lease.Token)
346+
}
306347
}
307-
topP := 0.95
308-
if req.TopP != nil {
309-
topP = *req.TopP
348+
349+
if lastErr != nil {
350+
return nil, lastErr
310351
}
311-
err := s.runGrokChatOnce(cw, r, lease, spec, message, fileInputs, temp, topP, emitThink, false, req.Model)
352+
return nil, platform.UpstreamError("Image generation completed but no image URL was returned (may be rate-limited or moderated)", 502, "")
353+
}
354+
355+
// captureImageURLsOnce executes one streaming chat attempt and collects image URLs.
356+
func (s *Server) captureImageURLsOnce(r *http.Request, lease *account.Lease, spec *model.Spec, message string, fileInputs []string) ([]string, error) {
357+
payload := grok.BuildChatPayload(message, model.ModeId(lease.ModeID), fileInputs, nil, nil, nil)
358+
body, err := json.Marshal(payload)
312359
if err != nil {
313-
return nil
360+
return nil, platform.UpstreamError("encode payload: "+err.Error(), 500, "")
314361
}
315362

316-
var obj map[string]any
317-
if err := json.Unmarshal(cw.body, &obj); err != nil {
318-
return nil
363+
ctx, cancel := context.WithTimeout(r.Context(), 5*time.Minute)
364+
defer cancel()
365+
366+
bodyReader, err := s.Transport.PostStream(ctx, grok.Chat, lease.Token, body)
367+
if err != nil {
368+
return nil, err
319369
}
320-
choices, _ := obj["choices"].([]any)
321-
if len(choices) == 0 {
322-
return nil
370+
defer bodyReader.Close()
371+
372+
adapter := grok.NewStreamAdapter()
373+
var imageURLs []string
374+
375+
scanner := bufio.NewScanner(bodyReader)
376+
scanner.Buffer(make([]byte, 64*1024), 4*1024*1024)
377+
for scanner.Scan() {
378+
line := scanner.Text()
379+
kind, data := grok.ClassifyLine(line)
380+
if kind == "done" {
381+
break
382+
}
383+
if kind != "data" {
384+
continue
385+
}
386+
events, _ := adapter.Feed([]byte(data))
387+
for _, ev := range events {
388+
if ev.Kind == grok.EventImage && ev.Content != "" {
389+
url := ev.Content
390+
if !strings.HasPrefix(url, "http") {
391+
url = grok.ImageBaseURL + strings.TrimPrefix(url, "/")
392+
}
393+
imageURLs = append(imageURLs, url)
394+
}
395+
}
323396
}
324-
choice, _ := choices[0].(map[string]any)
325-
msg, _ := choice["message"].(map[string]any)
326-
if msg == nil {
327-
return nil
397+
398+
// Also collect any URLs from the adapter's ImageURLs accumulator.
399+
for _, pair := range adapter.ImageURLs {
400+
url := pair[0]
401+
if url != "" {
402+
found := false
403+
for _, existing := range imageURLs {
404+
if existing == url {
405+
found = true
406+
break
407+
}
408+
}
409+
if !found {
410+
if !strings.HasPrefix(url, "http") {
411+
url = grok.ImageBaseURL + strings.TrimPrefix(url, "/")
412+
}
413+
imageURLs = append(imageURLs, url)
414+
}
415+
}
328416
}
329-
text, _ := msg["content"].(string)
330-
return extractImageURLsFromMarkdown(text)
417+
418+
return imageURLs, nil
331419
}
332420

333421
// captureLiteImageBatch runs N concurrent chat-based image generation
334-
// requests and returns all collected image URLs.
335-
func (s *Server) captureLiteImageBatch(r *http.Request, spec *model.Spec, prompt string, n int) []string {
422+
// requests and returns all collected image URLs and any error.
423+
func (s *Server) captureLiteImageBatch(r *http.Request, spec *model.Spec, prompt string, n int, ssoToken string) ([]string, error) {
336424
if n <= 0 {
337425
n = 1
338426
}
339427
results := make([]string, n)
340428
var wg sync.WaitGroup
429+
var firstErr error
430+
var errMu sync.Mutex
341431

342432
for i := 0; i < n; i++ {
343433
wg.Add(1)
@@ -348,10 +438,17 @@ func (s *Server) captureLiteImageBatch(r *http.Request, spec *model.Spec, prompt
348438
Model: spec.ModelName,
349439
Messages: msgs,
350440
}
351-
urls := s.captureImageURLs(r, chatReq, spec)
441+
urls, err := s.captureImageURLs(r, chatReq, spec, ssoToken)
352442
if len(urls) > 0 {
353443
results[idx] = urls[0]
354444
}
445+
if err != nil {
446+
errMu.Lock()
447+
if firstErr == nil {
448+
firstErr = err
449+
}
450+
errMu.Unlock()
451+
}
355452
}(i)
356453
}
357454
wg.Wait()
@@ -363,7 +460,10 @@ func (s *Server) captureLiteImageBatch(r *http.Request, spec *model.Spec, prompt
363460
out = append(out, u)
364461
}
365462
}
366-
return out
463+
if len(out) == 0 && firstErr != nil {
464+
return nil, firstErr
465+
}
466+
return out, nil
367467
}
368468

369469
// extractImageURLsFromMarkdown returns URLs found in markdown image syntax.
@@ -395,6 +495,24 @@ func fetchImageBase64(url string) (string, error) {
395495
return base64.StdEncoding.EncodeToString(body), nil
396496
}
397497

498+
// fetchImageBase64ViaTransport downloads the image bytes via the authenticated
499+
// Transport (carries cf_clearance and grok session cookies), then returns the
500+
// base64 encoding. This is needed for assets.grok.com URLs that require auth.
501+
func (s *Server) fetchImageBase64ViaTransport(url string) (string, error) {
502+
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
503+
defer cancel()
504+
bodyReader, err := s.Transport.GetBytes(ctx, url, "")
505+
if err != nil {
506+
return "", err
507+
}
508+
defer bodyReader.Close()
509+
body, err := io.ReadAll(io.LimitReader(bodyReader, 50<<20))
510+
if err != nil {
511+
return "", err
512+
}
513+
return base64.StdEncoding.EncodeToString(body), nil
514+
}
515+
398516
// --- Video jobs (async) ---
399517

400518
type videoJob struct {

0 commit comments

Comments
 (0)