From 4710959ab236ca8dffe61fb530863c753cacae2d Mon Sep 17 00:00:00 2001 From: Fabio Leitao Date: Tue, 29 Sep 2026 19:58:43 -0300 Subject: [PATCH] feat(providers): opt-in ZERO_RESPONSE_HEADER_TIMEOUT override for slow first-byte deployments The shared HTTP transport waits at most 120s for a response header. A local model server that has to load a large model can take longer than that to produce the first byte (measured 1-5 minutes on a throttled local Ollama), so an otherwise-alive request is aborted. Add ZERO_RESPONSE_HEADER_TIMEOUT, mirroring ZERO_STREAM_IDLE_TIMEOUT: a Go duration or a bare number of seconds; "0", "off", "none" or "disabled" remove the limit; an unparseable or non-positive value falls back to the default instead of removing the limit. The default stays 120s, so nothing changes for anyone who does not set the variable. The constant and the resolver sit next to the idle-timeout equivalents in providerio. Checked against a throttled local Ollama: ZERO_RESPONSE_HEADER_TIMEOUT=5s fails at ~6.4s, =300s succeeds at ~223s (a request that would have hit the 120s ceiling). Refs #1038 Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_0142QEjA7eEFcdXTpcTmcAZk --- internal/providers/providerio/providerio.go | 50 ++++++++++++-- .../response_header_timeout_resolve_test.go | 66 +++++++++++++++++++ 2 files changed, 109 insertions(+), 7 deletions(-) create mode 100644 internal/providers/providerio/response_header_timeout_resolve_test.go diff --git a/internal/providers/providerio/providerio.go b/internal/providers/providerio/providerio.go index e1adeb5cd..b24b34543 100644 --- a/internal/providers/providerio/providerio.go +++ b/internal/providers/providerio/providerio.go @@ -114,6 +114,46 @@ func ResolveStreamIdleTimeout(option time.Duration) time.Duration { return DefaultStreamIdleTimeout } +// DefaultResponseHeaderTimeout is how long the shared HTTP transport waits for +// a response header after the request is written. 120s, not 60s: a slow cloud +// proxy (e.g. ollama `*:cloud`) can withhold its 200 response header until the +// upstream model emits a first token, so a 60s cap risked aborting a +// legitimately-slow-but-alive request. 120s still bounds a truly dead reused +// connection (which never responds) while tolerating slow header delivery; slow +// first tokens after the header are covered by the idle + content-stall +// watchdogs. +const DefaultResponseHeaderTimeout = 120 * time.Second + +// responseHeaderTimeoutEnv is the global override for the response header +// timeout. It accepts the same forms as ZERO_STREAM_IDLE_TIMEOUT: a Go duration +// ("5m", "300s", "90s") or a bare number of seconds ("300"). A value of "0", +// "off", "none", or "disabled" removes the limit entirely (a connection that +// never answers may then wait until the request context ends). Useful when a +// local model server needs longer than DefaultResponseHeaderTimeout to produce +// the first byte, for example a cold model load on a throttled Ollama. +const responseHeaderTimeoutEnv = "ZERO_RESPONSE_HEADER_TIMEOUT" + +// ResolveResponseHeaderTimeout selects the effective response header timeout: +// the ZERO_RESPONSE_HEADER_TIMEOUT env override if set and valid, otherwise +// DefaultResponseHeaderTimeout. A returned value <= 0 means no limit. +func ResolveResponseHeaderTimeout() time.Duration { + if raw := strings.TrimSpace(os.Getenv(responseHeaderTimeoutEnv)); raw != "" { + switch strings.ToLower(raw) { + case "0", "off", "none", "disabled": + return 0 + } + if d, err := time.ParseDuration(raw); err == nil && d > 0 { + return d + } + if secs, err := strconv.Atoi(raw); err == nil && secs > 0 { + return time.Duration(secs) * time.Second + } + // Unparseable / non-positive: fall through to the default rather than + // silently removing the limit on a typo. + } + return DefaultResponseHeaderTimeout +} + // NormalizeBaseURL trims trailing slashes and validates an HTTP API base URL. func NormalizeBaseURL(baseURL string, defaultBaseURL string, label string) (string, error) { baseURL = strings.TrimSpace(baseURL) @@ -161,13 +201,9 @@ func NormalizeBaseURL(baseURL string, defaultBaseURL string, label string) (stri // pooled connections around as long. var sharedHTTPClient = func() *http.Client { transport := http.DefaultTransport.(*http.Transport).Clone() - // 120s, not 60s: a slow cloud proxy (e.g. ollama `*:cloud`) can withhold its - // 200 response header until the upstream model emits a first token, so a 60s cap - // risked aborting a legitimately-slow-but-alive request. 120s still bounds a - // truly dead reused connection (which never responds) while tolerating slow - // header delivery; slow first tokens after the header are covered by the idle + - // content-stall watchdogs. - transport.ResponseHeaderTimeout = 120 * time.Second + // DefaultResponseHeaderTimeout (120s) unless ZERO_RESPONSE_HEADER_TIMEOUT + // overrides it; see the constant for why the default is 120s and not 60s. + transport.ResponseHeaderTimeout = ResolveResponseHeaderTimeout() transport.IdleConnTimeout = 30 * time.Second // Periodically close idle connections to prevent stale HTTP/2 // connections from causing PROTOCOL_ERROR on the next request. diff --git a/internal/providers/providerio/response_header_timeout_resolve_test.go b/internal/providers/providerio/response_header_timeout_resolve_test.go new file mode 100644 index 000000000..712caa336 --- /dev/null +++ b/internal/providers/providerio/response_header_timeout_resolve_test.go @@ -0,0 +1,66 @@ +package providerio + +import ( + "testing" + "time" +) + +func TestResolveResponseHeaderTimeout(t *testing.T) { + const env = "ZERO_RESPONSE_HEADER_TIMEOUT" + + t.Run("default when env is unset or empty", func(t *testing.T) { + t.Setenv(env, "") + if got := ResolveResponseHeaderTimeout(); got != DefaultResponseHeaderTimeout { + t.Fatalf("got %v, want default %v", got, DefaultResponseHeaderTimeout) + } + }) + + t.Run("default keeps the value that was previously hardcoded", func(t *testing.T) { + if DefaultResponseHeaderTimeout != 120*time.Second { + t.Fatalf("default is %v; this override must not change the 120s default", DefaultResponseHeaderTimeout) + } + }) + + t.Run("env Go duration", func(t *testing.T) { + t.Setenv(env, "240s") + if got := ResolveResponseHeaderTimeout(); got != 240*time.Second { + t.Fatalf("got %v, want 240s", got) + } + t.Setenv(env, "5m") + if got := ResolveResponseHeaderTimeout(); got != 5*time.Minute { + t.Fatalf("got %v, want 5m", got) + } + }) + + t.Run("env bare seconds", func(t *testing.T) { + t.Setenv(env, "300") + if got := ResolveResponseHeaderTimeout(); got != 300*time.Second { + t.Fatalf("got %v, want 300s", got) + } + }) + + t.Run("env value is trimmed", func(t *testing.T) { + t.Setenv(env, " 90s ") + if got := ResolveResponseHeaderTimeout(); got != 90*time.Second { + t.Fatalf("got %v, want 90s", got) + } + }) + + t.Run("env removes the limit", func(t *testing.T) { + for _, value := range []string{"0", "off", "none", "disabled", "OFF", "Disabled"} { + t.Setenv(env, value) + if got := ResolveResponseHeaderTimeout(); got != 0 { + t.Fatalf("%q: got %v, want 0 (no limit)", value, got) + } + } + }) + + t.Run("invalid env falls back to default, not unlimited", func(t *testing.T) { + for _, value := range []string{"banana", "-5s", "-1", "1.5x"} { + t.Setenv(env, value) + if got := ResolveResponseHeaderTimeout(); got != DefaultResponseHeaderTimeout { + t.Fatalf("%q: got %v, want default %v (a typo must not remove the limit)", value, got, DefaultResponseHeaderTimeout) + } + } + }) +}