From 6060f99ea0bdab35d0f2639c71e8f5259a429f49 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Fri, 2 Oct 2026 21:48:42 +0800 Subject: [PATCH 01/12] update --- .gitignore | 5 - README.en.md | 4 +- README.fr.md | 4 +- README.ja.md | 4 +- README.md | 4 +- README.zh_CN.md | 4 +- README.zh_TW.md | 4 +- common/channel_health_observation.go | 154 +++ common/channel_health_observation_test.go | 85 ++ controller/channel-test.go | 146 ++- controller/channel_health_recovery_test.go | 81 ++ controller/smart_ops_alert.go | 91 ++ model/ability.go | 30 +- model/channel_cache.go | 55 +- model/channel_health_routing.go | 89 ++ model/channel_health_routing_test.go | 184 +++ model/option.go | 5 + pkg/channelhealth/runtime.go | 649 +++++++++++ pkg/channelhealth/runtime_test.go | 411 +++++++ relay/audio_handler.go | 3 + relay/channel/api_request.go | 52 + relay/channel/aws/relay-aws.go | 2 +- .../channel/cloudflare/first_response_test.go | 94 ++ relay/channel/cloudflare/relay_cloudflare.go | 18 +- relay/channel/cohere/first_response_test.go | 82 ++ relay/channel/cohere/relay-cohere.go | 16 +- relay/channel/openai/relay-openai.go | 2 +- relay/channel/openai/relay_image.go | 6 +- relay/claude_handler.go | 3 + .../common/channel_health_observation_test.go | 40 + relay/common/relay_info.go | 187 ++- relay/compatible_handler.go | 3 + relay/embedding_handler.go | 3 + relay/gemini_handler.go | 6 + .../stream_first_result_timeout_test.go | 73 ++ relay/helper/stream_result.go | 35 +- relay/helper/stream_scanner.go | 63 +- relay/image_handler.go | 3 + relay/relay_task.go | 3 + relay/rerank_handler.go | 3 + relay/responses_handler.go | 3 + router/api-router.go | 3 + service/smart_ops_alert.go | 520 ++++++++- service/smart_ops_alert_delivery_test.go | 183 +++ service/smart_ops_alert_test.go | 94 ++ setting/operation_setting/monitor_setting.go | 133 ++- .../operation_setting/monitor_setting_test.go | 36 + .../layout/config/smart-ops.config.test.ts | 8 +- .../layout/config/smart-ops.config.ts | 7 +- web/default/src/features/smart-ops/alerts.tsx | 21 +- web/default/src/features/smart-ops/api.ts | 33 + .../smart-ops/channel-health/index.tsx | 1009 +++++++++++++++++ .../src/features/smart-ops/lib/format.ts | 39 + web/default/src/features/smart-ops/types.ts | 30 +- web/default/src/i18n/locales/en.json | 86 +- web/default/src/i18n/locales/fr.json | 81 +- web/default/src/i18n/locales/ja.json | 81 +- web/default/src/i18n/locales/ru.json | 81 +- web/default/src/i18n/locales/vi.json | 81 +- web/default/src/i18n/locales/zh.json | 86 +- web/default/src/routeTree.gen.ts | 23 + .../smart-ops/channel-health.tsx | 34 + 62 files changed, 5240 insertions(+), 138 deletions(-) create mode 100644 common/channel_health_observation.go create mode 100644 common/channel_health_observation_test.go create mode 100644 controller/channel_health_recovery_test.go create mode 100644 model/channel_health_routing.go create mode 100644 model/channel_health_routing_test.go create mode 100644 pkg/channelhealth/runtime.go create mode 100644 pkg/channelhealth/runtime_test.go create mode 100644 relay/channel/cloudflare/first_response_test.go create mode 100644 relay/channel/cohere/first_response_test.go create mode 100644 relay/common/channel_health_observation_test.go create mode 100644 relay/helper/stream_first_result_timeout_test.go create mode 100644 service/smart_ops_alert_delivery_test.go create mode 100644 setting/operation_setting/monitor_setting_test.go create mode 100644 web/default/src/features/smart-ops/channel-health/index.tsx create mode 100644 web/default/src/routes/_authenticated/smart-ops/channel-health.tsx diff --git a/.gitignore b/.gitignore index eb461a92..6b403e4d 100644 --- a/.gitignore +++ b/.gitignore @@ -11,11 +11,6 @@ logs web/default/dist web/node_modules .env -one-api -max-api -new-api -sub2api -CLIProxyAPI /__debug_bin* .DS_Store tiktoken_cache diff --git a/README.en.md b/README.en.md index 45f7306b..c3600849 100644 --- a/README.en.md +++ b/README.en.md @@ -75,7 +75,7 @@ MAX API is more than a code repository. It is a long-term open collaboration aro | [MAX API Issues](https://github.com/MAX-API-Next/MAX-API/issues) | Report reproducible problems, propose improvements, and share compatibility changes | | [MAX API Releases](https://github.com/MAX-API-Next/MAX-API/releases) | Follow stable release updates | | [Ask DeepWiki](https://deepwiki.com/MAX-API-Next/MAX-API) | Search and understand the codebase quickly | -| Technical and ecosystem cooperation | Contact `maxapi@max-api.ai` | +| Technical and ecosystem cooperation | Contact `cscitech@cscitech.top` | ### Contributors we are looking for @@ -345,7 +345,7 @@ Multi-node deployments must share `SESSION_SECRET` while using a different `NODE This project is licensed under the [GNU Affero General Public License v3.0 (AGPLv3)](./LICENSE). -If you modify this project and provide it to users over a network, please understand and comply with AGPLv3 source-availability obligations. For institutional cooperation or other licensing questions, contact maxapi@max-api.ai. +If you modify this project and provide it to users over a network, please understand and comply with AGPLv3 source-availability obligations. For institutional cooperation or other licensing questions, contact cscitech@cscitech.top. --- diff --git a/README.fr.md b/README.fr.md index 0f0446ee..e1d019e9 100644 --- a/README.fr.md +++ b/README.fr.md @@ -75,7 +75,7 @@ MAX API est plus qu'un dépôt de code. C'est une collaboration ouverte de long | [Issues MAX API](https://github.com/MAX-API-Next/MAX-API/issues) | Signaler un problème reproductible, proposer une amélioration ou partager un changement de compatibilité | | [Versions MAX API](https://github.com/MAX-API-Next/MAX-API/releases) | Suivre les mises à jour des versions stables | | [Ask DeepWiki](https://deepwiki.com/MAX-API-Next/MAX-API) | Rechercher et comprendre rapidement le code | -| Coopération technique et écosystème | Contacter `maxapi@max-api.ai` | +| Coopération technique et écosystème | Contacter `cscitech@cscitech.top` | ### Profils de contributeurs recherchés @@ -345,7 +345,7 @@ Les déploiements multi-nœuds doivent partager `SESSION_SECRET`, tout en utilis Ce projet est distribué sous la [GNU Affero General Public License v3.0 (AGPLv3)](./LICENSE). -Si vous modifiez ce projet et le fournissez à des utilisateurs via un réseau, veuillez comprendre et respecter les obligations de mise à disposition du code source prévues par l'AGPLv3. Pour une coopération institutionnelle ou toute question de licence, contactez maxapi@max-api.ai. +Si vous modifiez ce projet et le fournissez à des utilisateurs via un réseau, veuillez comprendre et respecter les obligations de mise à disposition du code source prévues par l'AGPLv3. Pour une coopération institutionnelle ou toute question de licence, contactez cscitech@cscitech.top. --- diff --git a/README.ja.md b/README.ja.md index d33669d6..a7d6d130 100644 --- a/README.ja.md +++ b/README.ja.md @@ -75,7 +75,7 @@ MAX API は単なるコードリポジトリではありません。AI Models、 | [MAX API Issues](https://github.com/MAX-API-Next/MAX-API/issues) | 再現可能な問題、改善提案、互換性の変化を共有 | | [MAX API Releases](https://github.com/MAX-API-Next/MAX-API/releases) | 正式版の更新を確認 | | [Ask DeepWiki](https://deepwiki.com/MAX-API-Next/MAX-API) | コードベースをすばやく検索・理解 | -| 技術・エコシステム連携 | `maxapi@max-api.ai` まで連絡 | +| 技術・エコシステム連携 | `cscitech@cscitech.top` まで連絡 | ### 求めている貢献者 @@ -345,7 +345,7 @@ docker compose up -d 本プロジェクトは [GNU Affero General Public License v3.0（AGPLv3）](./LICENSE) の下で提供されます。 -本プロジェクトを変更し、ネットワーク経由でユーザーへ提供する場合は、AGPLv3 のソースコード提供義務を理解し、遵守してください。機関との協力、その他のライセンスに関する問い合わせは maxapi@max-api.ai までご連絡ください。 +本プロジェクトを変更し、ネットワーク経由でユーザーへ提供する場合は、AGPLv3 のソースコード提供義務を理解し、遵守してください。機関との協力、その他のライセンスに関する問い合わせは cscitech@cscitech.top までご連絡ください。 --- diff --git a/README.md b/README.md index 8cf8014f..1f215350 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ MAX API 不只是一个代码仓库，也是一项面向 AI Models、Agents、Ag | [MAX API Issues](https://github.com/MAX-API-Next/MAX-API/issues) | 提交可复现问题、需求建议和兼容性变化 | | [MAX API Releases](https://github.com/MAX-API-Next/MAX-API/releases) | 获取正式版本发布动态 | | [Ask DeepWiki](https://deepwiki.com/MAX-API-Next/MAX-API) | 快速检索和理解项目代码 | -| 技术与生态合作 | 联系 `maxapi@max-api.ai` | +| 技术与生态合作 | 联系 `cscitech@cscitech.top` | ### 我们正在寻找这些共建者 @@ -347,7 +347,7 @@ docker compose up -d 本项目采用 [GNU Affero 通用公共许可证 v3.0（AGPLv3）](./LICENSE) 授权。 -如果你修改并通过网络向用户提供本项目服务，请理解并遵守 AGPLv3 对应的源码提供等义务。机构合作或其他许可问题，请加入社群或联系：maxapi@max-api.ai。 +如果你修改并通过网络向用户提供本项目服务，请理解并遵守 AGPLv3 对应的源码提供等义务。机构合作或其他许可问题，请加入社群或联系：cscitech@cscitech.top。 --- diff --git a/README.zh_CN.md b/README.zh_CN.md index 7358bf3b..34ae42f4 100644 --- a/README.zh_CN.md +++ b/README.zh_CN.md @@ -75,7 +75,7 @@ MAX API 不只是一个代码仓库，也是一项面向 AI Models、Agents、Ag | [MAX API Issues](https://github.com/MAX-API-Next/MAX-API/issues) | 提交可复现问题、需求建议和兼容性变化 | | [MAX API Releases](https://github.com/MAX-API-Next/MAX-API/releases) | 获取正式版本发布动态 | | [Ask DeepWiki](https://deepwiki.com/MAX-API-Next/MAX-API) | 快速检索和理解项目代码 | -| 技术与生态合作 | 联系 `maxapi@max-api.ai` | +| 技术与生态合作 | 联系 `cscitech@cscitech.top` | ### 我们正在寻找这些共建者 @@ -347,7 +347,7 @@ docker compose up -d 本项目采用 [GNU Affero 通用公共许可证 v3.0（AGPLv3）](./LICENSE) 授权。 -如果你修改并通过网络向用户提供本项目服务，请理解并遵守 AGPLv3 对应的源码提供等义务。机构合作或其他许可问题，请联系：maxapi@max-api.ai。 +如果你修改并通过网络向用户提供本项目服务，请理解并遵守 AGPLv3 对应的源码提供等义务。机构合作或其他许可问题，请联系：cscitech@cscitech.top。 --- diff --git a/README.zh_TW.md b/README.zh_TW.md index 659fb306..32437e89 100644 --- a/README.zh_TW.md +++ b/README.zh_TW.md @@ -75,7 +75,7 @@ MAX API 不只是一個程式碼倉庫，也是一項面向 AI Models、Agents | [MAX API Issues](https://github.com/MAX-API-Next/MAX-API/issues) | 提交可重現問題、需求建議與相容性變化 | | [MAX API Releases](https://github.com/MAX-API-Next/MAX-API/releases) | 取得正式版本發布動態 | | [Ask DeepWiki](https://deepwiki.com/MAX-API-Next/MAX-API) | 快速檢索與理解專案程式碼 | -| 技術與生態合作 | 聯絡 `maxapi@max-api.ai` | +| 技術與生態合作 | 聯絡 `cscitech@cscitech.top` | ### 我們正在尋找這些共建者 @@ -347,7 +347,7 @@ docker compose up -d 本專案採用 [GNU Affero 通用公共許可證 v3.0（AGPLv3）](./LICENSE) 授權。 -如果你修改並透過網路向使用者提供本專案服務，請理解並遵守 AGPLv3 對應的原始碼提供等義務。機構合作或其他許可問題，請聯絡：maxapi@max-api.ai。 +如果你修改並透過網路向使用者提供本專案服務，請理解並遵守 AGPLv3 對應的原始碼提供等義務。機構合作或其他許可問題，請聯絡：cscitech@cscitech.top。 --- diff --git a/common/channel_health_observation.go b/common/channel_health_observation.go new file mode 100644 index 00000000..eb8fc1dc --- /dev/null +++ b/common/channel_health_observation.go @@ -0,0 +1,154 @@ +package common + +import ( + "context" + "errors" + "fmt" + "strconv" + "sync" + "time" + + "github.com/go-redis/redis/v8" +) + +const ( + channelHealthRuntimeKeyPrefix = "maxapi:channel-timeout:v1:observation:" + channelHealthRuntimeTTL = 24 * time.Hour + channelHealthObservationTimeout = 100 * time.Millisecond + channelHealthObservationQueueSize = 256 +) + +var channelHealthObservationQueue = make(chan ChannelHealthObservation, channelHealthObservationQueueSize) + +var channelHealthObservationWorkerOnce sync.Once + +func startChannelHealthObservationWorker() { + channelHealthObservationWorkerOnce.Do(func() { + go func() { + for observation := range channelHealthObservationQueue { + // The worker is deliberately best-effort. A slow or unavailable + // Redis instance must not remain on a relay response path. + _ = RecordChannelHealthObservation(observation) + } + }() + }) +} + +// EnqueueChannelHealthObservation schedules a best-effort diagnostic write. +// It never waits for Redis and drops the observation when the bounded queue is +// full, preserving the upstream response path as fail-open. +func EnqueueChannelHealthObservation(observation ChannelHealthObservation) { + if observation.ChannelID <= 0 || observation.Event == "" || !RedisEnabled || RDB == nil { + return + } + startChannelHealthObservationWorker() + select { + case channelHealthObservationQueue <- observation: + default: + // A full diagnostics queue is equivalent to an unavailable optional + // projection. The request path remains unaffected. + } +} + +// ChannelHealthObservation is the minimal, non-sensitive runtime fact emitted +// by a selected upstream channel attempt. It deliberately carries no request +// or response body and no provider credential. +type ChannelHealthObservation struct { + ChannelID int + Event string + ObservedAt time.Time + AttemptLatencyMS int64 + Stream bool + RetryIndex int +} + +// ChannelHealthRuntimeTransition is a small process-local bridge from the +// relay/runtime package to Smart Operations. The runtime state itself remains +// in Redis; this hook only lets the existing notification projector react +// immediately without introducing a package cycle. +type ChannelHealthRuntimeTransition struct { + ChannelID int + Transition string + Penalty int64 + PenaltyUntil time.Time + RuntimeDisabled bool + DisabledUntil time.Time + ObservedAt time.Time +} + +var channelHealthRuntimeTransitionObserver struct { + sync.RWMutex + fn func(ChannelHealthRuntimeTransition) +} + +func SetChannelHealthRuntimeTransitionObserver(observer func(ChannelHealthRuntimeTransition)) { + channelHealthRuntimeTransitionObserver.Lock() + channelHealthRuntimeTransitionObserver.fn = observer + channelHealthRuntimeTransitionObserver.Unlock() +} + +func PublishChannelHealthRuntimeTransition(transition ChannelHealthRuntimeTransition) { + channelHealthRuntimeTransitionObserver.RLock() + observer := channelHealthRuntimeTransitionObserver.fn + channelHealthRuntimeTransitionObserver.RUnlock() + if observer != nil { + observer(transition) + } +} + +func isChannelHealthTimeoutEvent(event string) bool { + switch event { + case "transport_timeout", "stream_idle_timeout", "request_timeout", + "streaming_first_result_timeout", "non_streaming_response_timeout": + return true + default: + return false + } +} + +func channelHealthRuntimeKey(channelID int) string { + return fmt.Sprintf("%s%d", channelHealthRuntimeKeyPrefix, channelID) +} + +// RecordChannelHealthObservation writes a bounded runtime projection. Redis +// failure is returned to the caller, which must treat it as diagnostic-only +// and never let it change relay behavior. +func RecordChannelHealthObservation(observation ChannelHealthObservation) error { + if observation.ChannelID <= 0 || observation.Event == "" { + return errors.New("channel health observation requires channel id and event") + } + if !RedisEnabled || RDB == nil { + return nil + } + if observation.ObservedAt.IsZero() { + observation.ObservedAt = time.Now() + } + if observation.AttemptLatencyMS < 0 { + observation.AttemptLatencyMS = 0 + } + + ctx, cancel := context.WithTimeout(context.Background(), channelHealthObservationTimeout) + defer cancel() + key := channelHealthRuntimeKey(observation.ChannelID) + fields := map[string]interface{}{ + "channel_id": strconv.Itoa(observation.ChannelID), + "last_event": observation.Event, + "last_observed_at": observation.ObservedAt.UTC().Format(time.RFC3339Nano), + "last_attempt_latency": strconv.FormatInt(observation.AttemptLatencyMS, 10), + "last_stream": strconv.FormatBool(observation.Stream), + "last_retry_index": strconv.Itoa(observation.RetryIndex), + } + + pipe := RDB.TxPipeline() + pipe.HIncrBy(ctx, key, "observations", 1) + if isChannelHealthTimeoutEvent(observation.Event) { + pipe.HIncrBy(ctx, key, "timeouts", 1) + } + pipe.HSet(ctx, key, fields) + pipe.Expire(ctx, key, channelHealthRuntimeTTL) + _, err := pipe.Exec(ctx) + if err != nil && !errors.Is(err, redis.Nil) { + return fmt.Errorf("record channel health observation: %w", err) + } + return nil +} diff --git a/common/channel_health_observation_test.go b/common/channel_health_observation_test.go new file mode 100644 index 00000000..18b1791f --- /dev/null +++ b/common/channel_health_observation_test.go @@ -0,0 +1,85 @@ +package common + +import ( + "context" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/go-redis/redis/v8" + "github.com/stretchr/testify/require" +) + +func TestRecordChannelHealthObservationStoresBoundedRuntimeProjection(t *testing.T) { + server := miniredis.RunT(t) + oldRDB, oldEnabled := RDB, RedisEnabled + RDB = redis.NewClient(&redis.Options{Addr: server.Addr()}) + RedisEnabled = true + t.Cleanup(func() { + _ = RDB.Close() + RDB, RedisEnabled = oldRDB, oldEnabled + }) + + now := time.Date(2026, 9, 30, 12, 0, 0, 0, time.UTC) + require.NoError(t, RecordChannelHealthObservation(ChannelHealthObservation{ + ChannelID: 936, + Event: "first_response", + ObservedAt: now, + AttemptLatencyMS: 1200, + Stream: true, + RetryIndex: 1, + })) + require.NoError(t, RecordChannelHealthObservation(ChannelHealthObservation{ + ChannelID: 936, + Event: "stream_idle_timeout", + ObservedAt: now.Add(time.Second), + AttemptLatencyMS: 2200, + Stream: true, + RetryIndex: 1, + })) + + fields, err := RDB.HGetAll(context.Background(), channelHealthRuntimeKey(936)).Result() + require.NoError(t, err) + require.Equal(t, "2", fields["observations"]) + require.Equal(t, "1", fields["timeouts"]) + require.Equal(t, "stream_idle_timeout", fields["last_event"]) + require.Equal(t, "2200", fields["last_attempt_latency"]) + require.Equal(t, "true", fields["last_stream"]) + require.Equal(t, "1", fields["last_retry_index"]) + require.Equal(t, "936", fields["channel_id"]) +} + +func TestRecordChannelHealthObservationCountsCanonicalTimeoutKinds(t *testing.T) { + server := miniredis.RunT(t) + oldRDB, oldEnabled := RDB, RedisEnabled + RDB = redis.NewClient(&redis.Options{Addr: server.Addr()}) + RedisEnabled = true + t.Cleanup(func() { + _ = RDB.Close() + RDB, RedisEnabled = oldRDB, oldEnabled + }) + + for _, event := range []string{"streaming_first_result_timeout", "non_streaming_response_timeout"} { + require.NoError(t, RecordChannelHealthObservation(ChannelHealthObservation{ + ChannelID: 936, + Event: event, + })) + } + fields, err := RDB.HGetAll(context.Background(), channelHealthRuntimeKey(936)).Result() + require.NoError(t, err) + require.Equal(t, "2", fields["timeouts"]) +} + +func TestRecordChannelHealthObservationSkipsRedisWhenDisabled(t *testing.T) { + oldRDB, oldEnabled := RDB, RedisEnabled + RDB = nil + RedisEnabled = false + t.Cleanup(func() { + RDB, RedisEnabled = oldRDB, oldEnabled + }) + + require.NoError(t, RecordChannelHealthObservation(ChannelHealthObservation{ + ChannelID: 1, + Event: "first_response", + })) +} diff --git a/controller/channel-test.go b/controller/channel-test.go index 5a11c7ea..cf86324a 100644 --- a/controller/channel-test.go +++ b/controller/channel-test.go @@ -2,6 +2,7 @@ package controller import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -21,6 +22,7 @@ import ( "github.com/MAX-API-Next/MAX-API/middleware" "github.com/MAX-API-Next/MAX-API/model" "github.com/MAX-API-Next/MAX-API/pkg/billingexpr" + "github.com/MAX-API-Next/MAX-API/pkg/channelhealth" "github.com/MAX-API-Next/MAX-API/relay" relaycommon "github.com/MAX-API-Next/MAX-API/relay/common" relayconstant "github.com/MAX-API-Next/MAX-API/relay/constant" @@ -982,6 +984,19 @@ func testChannels(channels []*model.Channel, testUserID int, notify bool, allowD continue } isChannelEnabled := channel.Status == common.ChannelStatusEnabled + setting := operation_setting.GetMonitorSetting() + var runtimeState channelhealth.RuntimeState + var hasRuntimeState bool + if setting.TimeoutAutoDisableEnabled { + runtimeState, hasRuntimeState = loadChannelHealthRuntimeState(channel.Id) + } + runtimeRecoveryProbe := hasRuntimeState && runtimeState.RuntimeDisabled && + (runtimeState.DisabledUntil.IsZero() || !time.Now().Before(runtimeState.DisabledUntil)) && + setting.TimeoutAutoDisableRecoveryMode == operation_setting.TimeoutAutoDisableRecoveryProbe && + setting.TimeoutAutoDisableDurationSeconds > 0 + if hasRuntimeState && !runtimeRecoveryProbe { + continue + } tik := time.Now() result := testChannel(channel, testUserID, "", "", shouldUseStreamForAutomaticChannelTest(channel)) tok := time.Now() @@ -1004,12 +1019,31 @@ func testChannels(channels []*model.Channel, testUserID int, notify bool, allowD } // disable channel - if allowDisable && isChannelEnabled && shouldBanChannel && channel.GetAutoBan() { + if allowDisable && !runtimeRecoveryProbe && isChannelEnabled && shouldBanChannel && channel.GetAutoBan() { processChannelError(result.context, *types.NewChannelError(channel.Id, channel.Type, channel.Name, channel.ChannelInfo.IsMultiKey, common.GetContextKeyString(result.context, constant.ContextKeyChannelKey), channel.GetAutoBan()), maxAPIError) } + if runtimeRecoveryProbe { + probeID := fmt.Sprintf("scheduled-channel-recovery:%d:%d", channel.Id, tik.UnixNano()) + probeSetting := operation_setting.GetMonitorSetting() + probeResult, probeErr := channelhealth.RecoverRuntimeState(context.Background(), channel.Id, probeID, result.localErr == nil && maxAPIError == nil, probeSetting) + if probeErr != nil { + common.SysLog(fmt.Sprintf("channel %d runtime recovery probe failed: %v", channel.Id, probeErr)) + } else if probeResult.Transition != "" { + common.PublishChannelHealthRuntimeTransition(common.ChannelHealthRuntimeTransition{ + ChannelID: channel.Id, + Transition: probeResult.Transition, + Penalty: probeResult.State.Penalty, + PenaltyUntil: probeResult.State.PenaltyUntil, + RuntimeDisabled: probeResult.State.RuntimeDisabled, + DisabledUntil: probeResult.State.DisabledUntil, + ObservedAt: probeResult.State.LastObservedAt, + }) + } + } + // enable channel - if result.localErr == nil && !isChannelEnabled && service.ShouldEnableChannel(maxAPIError, channel.Status) { + if shouldEnableLegacyChannelAfterTest(runtimeRecoveryProbe, result.localErr, isChannelEnabled, maxAPIError, channel.Status) { service.EnableChannel(channel.Id, common.GetContextKeyString(result.context, constant.ContextKeyChannelKey), channel.Name) } @@ -1024,6 +1058,12 @@ func testChannels(channels []*model.Channel, testUserID int, notify bool, allowD return nil } +// Runtime timeout recovery must not pass through the legacy SQL auto-enable +// path. The two state sources have independent ownership and recovery rules. +func shouldEnableLegacyChannelAfterTest(runtimeRecoveryProbe bool, localErr error, isChannelEnabled bool, maxAPIError *types.MaxAPIError, status int) bool { + return !runtimeRecoveryProbe && localErr == nil && !isChannelEnabled && service.ShouldEnableChannel(maxAPIError, status) +} + func selectChannelsForAutomaticTest(channels []*model.Channel, mode string) []*model.Channel { selected := make([]*model.Channel, 0, len(channels)) for _, channel := range channels { @@ -1059,7 +1099,97 @@ func testAutoDisabledChannels(notify bool) error { if getChannelErr != nil { return getChannelErr } - return testChannels(selectChannelsForAutomaticTest(channels, operation_setting.ChannelTestModePassiveRecovery), testUserID, notify, false) + return testChannels(selectChannelsForRuntimeRecovery(channels), testUserID, notify, false) +} + +// Runtime recovery remains scheduled when the legacy all-channel test is +// disabled. Only due, SQL-enabled runtime-disabled channels are probed. +func testRuntimeDisabledChannels() error { + channels, err := model.GetAllChannels(0, 0, true, false) + if err != nil { + return err + } + selected := selectChannelsForRuntimeRecovery(channels) + runtimeOnly := make([]*model.Channel, 0, len(selected)) + for _, channel := range selected { + if channel.Status == common.ChannelStatusEnabled { + runtimeOnly = append(runtimeOnly, channel) + } + } + if len(runtimeOnly) == 0 { + return nil + } + testUserID, err := resolveChannelTestUserID(nil) + if err != nil { + return err + } + return testChannels(runtimeOnly, testUserID, false, false) +} + +func loadChannelHealthRuntimeState(channelID int) (channelhealth.RuntimeState, bool) { + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + state, err := channelhealth.LoadRuntimeState(ctx, channelID) + if err != nil || !state.RuntimeDisabled { + return channelhealth.RuntimeState{}, false + } + return state, true +} + +func selectChannelsForRuntimeRecovery(channels []*model.Channel) []*model.Channel { + selected := make([]*model.Channel, 0, len(channels)) + now := time.Now() + setting := operation_setting.GetMonitorSetting() + for _, channel := range channels { + if channel == nil || channel.Status == common.ChannelStatusManuallyDisabled { + continue + } + if channel.Status == common.ChannelStatusAutoDisabled { + selected = append(selected, channel) + continue + } + if !setting.TimeoutAutoDisableEnabled || + setting.TimeoutAutoDisableRecoveryMode != operation_setting.TimeoutAutoDisableRecoveryProbe || + setting.TimeoutAutoDisableDurationSeconds <= 0 { + continue + } + state, ok := loadChannelHealthRuntimeState(channel.Id) + if ok && (state.DisabledUntil.IsZero() || !now.Before(state.DisabledUntil)) { + selected = append(selected, channel) + } + } + return selected +} + +// RecoverChannelHealthRuntime clears only the timeout strategy's Redis state. +// The existing channel status and ability rows are intentionally untouched. +func RecoverChannelHealthRuntime(c *gin.Context) { + channelID, err := strconv.Atoi(strings.TrimSpace(c.Param("id"))) + if err != nil || channelID <= 0 { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "无效的渠道 ID"}) + return + } + result, err := channelhealth.RestoreRuntimeState(c.Request.Context(), channelID) + if err != nil { + common.ApiError(c, err) + return + } + if result.Applied { + common.PublishChannelHealthRuntimeTransition(common.ChannelHealthRuntimeTransition{ + ChannelID: channelID, + Transition: result.Transition, + Penalty: result.State.Penalty, + PenaltyUntil: result.State.PenaltyUntil, + RuntimeDisabled: result.State.RuntimeDisabled, + DisabledUntil: result.State.DisabledUntil, + ObservedAt: result.State.LastObservedAt, + }) + recordManageAudit(c, "smart_ops.channel_health_runtime_recovery", map[string]interface{}{ + "channel_id": channelID, + "transition": result.Transition, + }) + } + c.JSON(http.StatusOK, gin.H{"success": true, "data": result.State}) } func TestAllChannels(c *gin.Context) { @@ -1083,7 +1213,15 @@ func AutomaticallyTestChannels() { } autoTestChannelsOnce.Do(func() { for { - if !operation_setting.GetMonitorSetting().AutoTestChannelEnabled { + setting := operation_setting.GetMonitorSetting() + if !setting.AutoTestChannelEnabled { + if setting.TimeoutAutoDisableEnabled && + setting.TimeoutAutoDisableRecoveryMode == operation_setting.TimeoutAutoDisableRecoveryProbe && + setting.TimeoutAutoDisableDurationSeconds > 0 { + if err := testRuntimeDisabledChannels(); err != nil { + common.SysLog(fmt.Sprintf("automatic channel runtime recovery failed: %v", err)) + } + } time.Sleep(1 * time.Minute) continue } diff --git a/controller/channel_health_recovery_test.go b/controller/channel_health_recovery_test.go new file mode 100644 index 00000000..27ccc30b --- /dev/null +++ b/controller/channel_health_recovery_test.go @@ -0,0 +1,81 @@ +package controller + +import ( + "context" + "testing" + "time" + + "github.com/MAX-API-Next/MAX-API/common" + "github.com/MAX-API-Next/MAX-API/model" + "github.com/MAX-API-Next/MAX-API/pkg/channelhealth" + "github.com/MAX-API-Next/MAX-API/setting/operation_setting" + "github.com/alicebob/miniredis/v2" + "github.com/go-redis/redis/v8" + "github.com/stretchr/testify/require" +) + +func TestSelectChannelsForRuntimeRecoveryOnlyProbesDueEligibleChannels(t *testing.T) { + server := miniredis.RunT(t) + previousRedis, previousEnabled := common.RDB, common.RedisEnabled + common.RDB = redis.NewClient(&redis.Options{Addr: server.Addr()}) + common.RedisEnabled = true + previousSetting := *operation_setting.GetMonitorSetting() + t.Cleanup(func() { + _ = common.RDB.Close() + common.RDB, common.RedisEnabled = previousRedis, previousEnabled + *operation_setting.GetMonitorSetting() = previousSetting + }) + setting := operation_setting.GetMonitorSetting() + setting.TimeoutAutoDisableEnabled = true + setting.TimeoutAutoDisableRecoveryMode = operation_setting.TimeoutAutoDisableRecoveryProbe + setting.TimeoutAutoDisableDurationSeconds = 60 + + now := time.Now() + for id, until := range map[int]time.Time{ + 1: now.Add(-time.Minute), + 2: now.Add(time.Hour), + 3: now.Add(-time.Minute), + } { + err := common.RDB.HSet(context.Background(), channelhealth.RuntimeStateKey(id), map[string]interface{}{ + "runtime_disabled": "true", + "disabled_until": until.UTC().Format(time.RFC3339Nano), + }).Err() + require.NoError(t, err) + } + channels := []*model.Channel{ + {Id: 1, Status: common.ChannelStatusEnabled}, + {Id: 2, Status: common.ChannelStatusEnabled}, + {Id: 3, Status: common.ChannelStatusManuallyDisabled}, + {Id: 4, Status: common.ChannelStatusAutoDisabled}, + } + selected := selectChannelsForRuntimeRecovery(channels) + require.Len(t, selected, 2) + require.Equal(t, 1, selected[0].Id) + require.Equal(t, 4, selected[1].Id) + + setting.TimeoutAutoDisableRecoveryMode = operation_setting.TimeoutAutoDisableRecoveryManual + selected = selectChannelsForRuntimeRecovery(channels) + require.Len(t, selected, 1) + require.Equal(t, 4, selected[0].Id) +} + +func TestRuntimeRecoveryDoesNotReenableLegacySQLAutoDisabledChannel(t *testing.T) { + previousAutomaticEnable := common.AutomaticEnableChannelEnabled + common.AutomaticEnableChannelEnabled = true + t.Cleanup(func() { common.AutomaticEnableChannelEnabled = previousAutomaticEnable }) + + require.False(t, shouldEnableLegacyChannelAfterTest( + true, + nil, + false, + nil, + common.ChannelStatusAutoDisabled, + )) + require.True(t, shouldEnableLegacyChannelAfterTest( + false, + nil, + false, + nil, + common.ChannelStatusAutoDisabled, + )) +} diff --git a/controller/smart_ops_alert.go b/controller/smart_ops_alert.go index 641c964a..db08365a 100644 --- a/controller/smart_ops_alert.go +++ b/controller/smart_ops_alert.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "strconv" + "strings" "github.com/MAX-API-Next/MAX-API/common" "github.com/MAX-API-Next/MAX-API/model" @@ -21,6 +22,96 @@ func GetSmartOpsAlerts(c *gin.Context) { }) } +var channelHealthPolicyKeys = []string{ + "monitor_setting.auto_priority_demotion_enabled", + "monitor_setting.streaming_first_result_timeout_seconds", + "monitor_setting.non_streaming_response_timeout_seconds", + "monitor_setting.priority_deduction", + "monitor_setting.timeout_auto_disable_enabled", + "monitor_setting.timeout_auto_disable_count", + "monitor_setting.timeout_auto_disable_window_seconds", + "monitor_setting.timeout_auto_disable_minimum_samples", + "monitor_setting.timeout_auto_disable_ratio_percent", + "monitor_setting.timeout_auto_disable_duration_seconds", + "monitor_setting.timeout_auto_disable_successful_probe_count", + "monitor_setting.timeout_auto_disable_recovery_mode", + "monitor_setting.timeout_auto_disable_count_scope", + "monitor_setting.penalty_cooldown_seconds", + "monitor_setting.recovery_mode", + "monitor_setting.extend_on_repeat_timeout", + "monitor_setting.channel_timeout_notification_enabled", + "monitor_setting.alert_repeat_interval_seconds", +} + +func isChannelHealthPolicyKey(key string) bool { + for _, allowed := range channelHealthPolicyKeys { + if key == allowed { + return true + } + } + return false +} + +func channelHealthPolicySnapshot() []model.Option { + common.OptionMapRWMutex.RLock() + defer common.OptionMapRWMutex.RUnlock() + options := make([]model.Option, 0, len(channelHealthPolicyKeys)) + for _, key := range channelHealthPolicyKeys { + options = append(options, model.Option{Key: key, Value: common.OptionMap[key]}) + } + return options +} + +// GetChannelHealthPolicy exposes only the non-sensitive monitor_setting keys +// needed by Smart Operations. It is intentionally available to administrators +// who can view the page; writes remain root-only. +func GetChannelHealthPolicy(c *gin.Context) { + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": channelHealthPolicySnapshot(), + }) +} + +type channelHealthPolicyUpdate struct { + Key string `json:"key"` + Value interface{} `json:"value"` +} + +type channelHealthPolicyUpdateRequest struct { + Updates []channelHealthPolicyUpdate `json:"updates"` +} + +// UpdateChannelHealthPolicy validates and saves all changed monitor_setting +// values in one existing-options transaction. No new persistence structure is +// introduced and a rejected value cannot leave a partial policy update. +func UpdateChannelHealthPolicy(c *gin.Context) { + var request channelHealthPolicyUpdateRequest + if err := common.DecodeJson(c.Request.Body, &request); err != nil || len(request.Updates) == 0 { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "updates is required"}) + return + } + values := make(map[string]string, len(request.Updates)) + for _, update := range request.Updates { + key := strings.TrimSpace(update.Key) + if !isChannelHealthPolicyKey(key) || !model.IsRegisteredOptionKey(key) { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "unsupported channel health option"}) + return + } + values[key] = common.Interface2String(update.Value) + } + if err := model.UpdateOptionsBulk(values); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": err.Error()}) + return + } + recordManageAudit(c, "smart_ops.channel_health_policy_update", map[string]interface{}{ + "keys": request.Updates, + }) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": channelHealthPolicySnapshot(), + }) +} + type billingSettlementBlockingPolicyRequest struct { BlockUserByDefault *bool `json:"block_user_by_default"` } diff --git a/model/ability.go b/model/ability.go index 0c8690b6..9be247e3 100644 --- a/model/ability.go +++ b/model/ability.go @@ -72,9 +72,16 @@ func GetChannel(group string, model string, retry int, requestPath string) (*Cha // GetChannelExcluding selects a channel while excluding channels already tried // during the current request's retry sequence. func GetChannelExcluding(group string, model string, retry int, requestPath string, excludedChannelIDs map[int]struct{}) (*Channel, error) { - channelQuery, err := getChannelQuery(group, model, retry) - if err != nil { - return nil, err + var channelQuery *gorm.DB + var err error + if channelHealthRoutingEnabled() { + channelQuery = DB.Model(&Ability{}). + Where(abilityCol("group")+" = ? and "+abilityCol("model")+" = ? and "+abilityCol("enabled")+" = ?", group, model, true) + } else { + channelQuery, err = getChannelQuery(group, model, retry) + if err != nil { + return nil, err + } } abilityRows, err := findAbilityRows(channelQuery) if err != nil { @@ -173,13 +180,14 @@ func selectChannelIdFromAbilities(abilities []Ability, retry int, excludedChanne if len(abilities) == 0 { return 0, false, nil } - priorities := make([]int64, 0, len(abilities)) + routingCandidates := buildAbilityRoutingCandidates(abilities) + priorities := make([]int64, 0, len(routingCandidates)) seenPriorities := make(map[int64]struct{}, len(abilities)) - for _, ability := range abilities { - if _, excluded := excludedChannelIDs[ability.ChannelId]; excluded { + for _, candidate := range routingCandidates { + if _, excluded := excludedChannelIDs[candidate.ability.ChannelId]; excluded { continue } - priority := abilityPriority(ability) + priority := candidate.priority if _, ok := seenPriorities[priority]; ok { continue } @@ -197,10 +205,10 @@ func selectChannelIdFromAbilities(abilities []Ability, retry int, excludedChanne } targetPriority := priorities[retry] - targetAbilities := make([]Ability, 0, len(abilities)) - for _, ability := range abilities { - if _, excluded := excludedChannelIDs[ability.ChannelId]; !excluded && abilityPriority(ability) == targetPriority { - targetAbilities = append(targetAbilities, ability) + targetAbilities := make([]Ability, 0, len(routingCandidates)) + for _, candidate := range routingCandidates { + if _, excluded := excludedChannelIDs[candidate.ability.ChannelId]; !excluded && candidate.priority == targetPriority { + targetAbilities = append(targetAbilities, candidate.ability) } } if len(targetAbilities) == 0 { diff --git a/model/channel_cache.go b/model/channel_cache.go index 751cbdf2..27029881 100644 --- a/model/channel_cache.go +++ b/model/channel_cache.go @@ -127,11 +127,11 @@ func GetRandomSatisfiedChannelExcluding(group string, model string, retry int, r } channelSyncLock.RLock() - defer channelSyncLock.RUnlock() // First, try to find channels with the exact model name. channels, err := filterChannelsByRequestPath(group2model2channels[group][model], requestPath) if err != nil { + channelSyncLock.RUnlock() return nil, err } @@ -140,11 +140,13 @@ func GetRandomSatisfiedChannelExcluding(group string, model string, retry int, r normalizedModel := ratio_setting.FormatMatchingModelName(model) channels, err = filterChannelsByRequestPath(group2model2channels[group][normalizedModel], requestPath) if err != nil { + channelSyncLock.RUnlock() return nil, err } } if len(channels) == 0 { + channelSyncLock.RUnlock() return nil, nil } if len(excludedChannelIDs) > 0 { @@ -157,46 +159,53 @@ func GetRandomSatisfiedChannelExcluding(group string, model string, retry int, r channels = availableChannels } if len(channels) == 0 { + channelSyncLock.RUnlock() return nil, nil } - if len(channels) == 1 { - if channel, ok := channelsIDM[channels[0]]; ok { - return channel, nil + cachedChannels := make([]*Channel, 0, len(channels)) + for _, channelID := range channels { + channel, ok := channelsIDM[channelID] + if !ok { + channelSyncLock.RUnlock() + return nil, fmt.Errorf("数据库一致性错误，渠道# %d 不存在，请联系管理员修复", channelID) } - return nil, fmt.Errorf("数据库一致性错误，渠道# %d 不存在，请联系管理员修复", channels[0]) + cachedChannels = append(cachedChannels, channel) } + channelSyncLock.RUnlock() - uniquePriorities := make(map[int]bool) - for _, channelId := range channels { - if channel, ok := channelsIDM[channelId]; ok { - uniquePriorities[int(channel.GetPriority())] = true - } else { - return nil, fmt.Errorf("数据库一致性错误，渠道# %d 不存在，请联系管理员修复", channelId) - } + candidates := buildChannelRoutingCandidates(cachedChannels) + if len(candidates) == 0 { + return nil, nil + } + if len(candidates) == 1 { + return candidates[0].channel, nil + } + + uniquePriorities := make(map[int64]bool) + for _, candidate := range candidates { + uniquePriorities[candidate.priority] = true } - var sortedUniquePriorities []int + var sortedUniquePriorities []int64 for priority := range uniquePriorities { sortedUniquePriorities = append(sortedUniquePriorities, priority) } - sort.Sort(sort.Reverse(sort.IntSlice(sortedUniquePriorities))) + sort.Slice(sortedUniquePriorities, func(i, j int) bool { + return sortedUniquePriorities[i] > sortedUniquePriorities[j] + }) if retry >= len(uniquePriorities) { retry = len(uniquePriorities) - 1 } - targetPriority := int64(sortedUniquePriorities[retry]) + targetPriority := sortedUniquePriorities[retry] // get the priority for the given retry number var sumWeight = 0 var targetChannels []*Channel - for _, channelId := range channels { - if channel, ok := channelsIDM[channelId]; ok { - if channel.GetPriority() == targetPriority { - sumWeight += channel.GetWeight() - targetChannels = append(targetChannels, channel) - } - } else { - return nil, fmt.Errorf("数据库一致性错误，渠道# %d 不存在，请联系管理员修复", channelId) + for _, candidate := range candidates { + if candidate.priority == targetPriority { + sumWeight += candidate.channel.GetWeight() + targetChannels = append(targetChannels, candidate.channel) } } diff --git a/model/channel_health_routing.go b/model/channel_health_routing.go new file mode 100644 index 00000000..7831917d --- /dev/null +++ b/model/channel_health_routing.go @@ -0,0 +1,89 @@ +package model + +import ( + "context" + "time" + + "github.com/MAX-API-Next/MAX-API/pkg/channelhealth" + "github.com/MAX-API-Next/MAX-API/setting/operation_setting" +) + +type channelRoutingCandidate struct { + channel *Channel + priority int64 +} + +func channelHealthRoutingEnabled() bool { + setting := operation_setting.GetMonitorSetting() + return setting != nil && (setting.AutoPriorityDemotionEnabled || setting.TimeoutAutoDisableEnabled) +} + +func loadChannelRuntimeStates(channelIDs []int) map[int]channelhealth.RuntimeState { + if !channelHealthRoutingEnabled() || len(channelIDs) == 0 { + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + states, err := channelhealth.LoadRuntimeStates(ctx, channelIDs) + if err != nil { + // Redis is an optional runtime dependency for this feature. The caller + // must keep the existing base-priority selection when it is unavailable. + return nil + } + return states +} + +func buildChannelRoutingCandidates(channels []*Channel) []channelRoutingCandidate { + if len(channels) == 0 { + return nil + } + channelIDs := make([]int, 0, len(channels)) + for _, channel := range channels { + if channel != nil { + channelIDs = append(channelIDs, channel.Id) + } + } + states := loadChannelRuntimeStates(channelIDs) + now := time.Now() + candidates := make([]channelRoutingCandidate, 0, len(channels)) + for _, channel := range channels { + if channel == nil { + continue + } + priority := channel.GetPriority() + if state, ok := states[channel.Id]; ok { + if state.RuntimeDisabled { + continue + } + priority = channelhealth.EffectivePriority(priority, state, now) + } + candidates = append(candidates, channelRoutingCandidate{channel: channel, priority: priority}) + } + return candidates +} + +func buildAbilityRoutingCandidates(abilities []Ability) []abilityRoutingCandidate { + channelIDs := make([]int, 0, len(abilities)) + for _, ability := range abilities { + channelIDs = append(channelIDs, ability.ChannelId) + } + states := loadChannelRuntimeStates(channelIDs) + now := time.Now() + candidates := make([]abilityRoutingCandidate, 0, len(abilities)) + for _, ability := range abilities { + priority := abilityPriority(ability) + if state, ok := states[ability.ChannelId]; ok { + if state.RuntimeDisabled { + continue + } + priority = channelhealth.EffectivePriority(priority, state, now) + } + candidates = append(candidates, abilityRoutingCandidate{ability: ability, priority: priority}) + } + return candidates +} + +type abilityRoutingCandidate struct { + ability Ability + priority int64 +} diff --git a/model/channel_health_routing_test.go b/model/channel_health_routing_test.go new file mode 100644 index 00000000..be874e0a --- /dev/null +++ b/model/channel_health_routing_test.go @@ -0,0 +1,184 @@ +package model + +import ( + "context" + "testing" + "time" + + maxcommon "github.com/MAX-API-Next/MAX-API/common" + "github.com/MAX-API-Next/MAX-API/pkg/channelhealth" + "github.com/MAX-API-Next/MAX-API/setting/operation_setting" + "github.com/alicebob/miniredis/v2" + "github.com/go-redis/redis/v8" + "github.com/stretchr/testify/require" +) + +func withChannelRoutingTestState(t *testing.T, setting operation_setting.MonitorSetting) { + t.Helper() + oldSetting := *operation_setting.GetMonitorSetting() + oldRDB, oldEnabled := maxcommon.RDB, maxcommon.RedisEnabled + server := miniredis.RunT(t) + maxcommon.RDB = redis.NewClient(&redis.Options{Addr: server.Addr()}) + maxcommon.RedisEnabled = true + *operation_setting.GetMonitorSetting() = setting + t.Cleanup(func() { + _ = maxcommon.RDB.Close() + maxcommon.RDB, maxcommon.RedisEnabled = oldRDB, oldEnabled + *operation_setting.GetMonitorSetting() = oldSetting + }) +} + +func TestAbilityRoutingUsesEffectivePriorityWithoutChangingBasePriority(t *testing.T) { + setting := operation_setting.MonitorSetting{ + AutoPriorityDemotionEnabled: true, + StreamingFirstResultTimeoutSeconds: 1, + PriorityDeduction: 20, + PenaltyCooldownSeconds: 60, + RecoveryMode: operation_setting.RecoveryModeAutomatic, + } + withChannelRoutingTestState(t, setting) + _, err := channelhealth.EvaluateTimeoutGuard(context.Background(), channelhealth.AttemptEvidence{ + ChannelID: 1, + AttemptID: "routing-penalty", + RequestMode: channelhealth.RequestModeStreaming, + TimeoutKind: channelhealth.TimeoutKindStreamingFirstResult, + TimeoutEligible: true, + AutoBan: true, + ObservedAt: time.Now(), + }, &setting) + require.NoError(t, err) + + baseHigh, baseLow := int64(100), int64(90) + abilities := []Ability{ + {ChannelId: 1, Priority: &baseHigh, Weight: 100}, + {ChannelId: 2, Priority: &baseLow, Weight: 100}, + } + selected, ok, err := selectChannelIdFromAbilities(abilities, 0, nil) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, 2, selected) + selected, ok, err = selectChannelIdFromAbilities(abilities, 1, nil) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, 1, selected) + require.Equal(t, int64(100), *abilities[0].Priority) +} + +func TestAbilityRoutingExcludesRuntimeDisabledChannel(t *testing.T) { + setting := operation_setting.MonitorSetting{ + TimeoutAutoDisableEnabled: true, + StreamingFirstResultTimeoutSeconds: 1, + TimeoutAutoDisableCount: 2, + TimeoutAutoDisableWindowSeconds: 3600, + TimeoutAutoDisableDurationSeconds: 1800, + TimeoutAutoDisableRecoveryMode: operation_setting.TimeoutAutoDisableRecoveryProbe, + } + withChannelRoutingTestState(t, setting) + for _, attemptID := range []string{"routing-disable-1", "routing-disable-2"} { + _, err := channelhealth.EvaluateTimeoutGuard(context.Background(), channelhealth.AttemptEvidence{ + ChannelID: 1, + AttemptID: attemptID, + RequestMode: channelhealth.RequestModeStreaming, + TimeoutKind: channelhealth.TimeoutKindStreamingFirstResult, + TimeoutEligible: true, + AutoBan: true, + }, &setting) + require.NoError(t, err) + } + + baseHigh, baseLow := int64(100), int64(90) + selected, ok, err := selectChannelIdFromAbilities([]Ability{ + {ChannelId: 1, Priority: &baseHigh, Weight: 100}, + {ChannelId: 2, Priority: &baseLow, Weight: 100}, + }, 0, nil) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, 2, selected) +} + +func TestAbilityRoutingFailsOpenWhenRedisUnavailable(t *testing.T) { + oldSetting := *operation_setting.GetMonitorSetting() + oldRDB, oldEnabled := maxcommon.RDB, maxcommon.RedisEnabled + t.Cleanup(func() { + maxcommon.RDB, maxcommon.RedisEnabled = oldRDB, oldEnabled + *operation_setting.GetMonitorSetting() = oldSetting + }) + *operation_setting.GetMonitorSetting() = operation_setting.MonitorSetting{ + AutoPriorityDemotionEnabled: true, + StreamingFirstResultTimeoutSeconds: 1, + PriorityDeduction: 20, + } + maxcommon.RDB = nil + maxcommon.RedisEnabled = false + + baseHigh, baseLow := int64(100), int64(90) + selected, ok, err := selectChannelIdFromAbilities([]Ability{ + {ChannelId: 1, Priority: &baseHigh, Weight: 100}, + {ChannelId: 2, Priority: &baseLow, Weight: 100}, + }, 0, nil) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, 1, selected) +} + +func TestAbilityRoutingFailsOpenWhenRedisDisconnects(t *testing.T) { + oldSetting := *operation_setting.GetMonitorSetting() + oldRDB, oldEnabled := maxcommon.RDB, maxcommon.RedisEnabled + client := redis.NewClient(&redis.Options{ + Addr: "127.0.0.1:1", + MaxRetries: 0, + DialTimeout: 20 * time.Millisecond, + ReadTimeout: 20 * time.Millisecond, + WriteTimeout: 20 * time.Millisecond, + }) + maxcommon.RDB = client + maxcommon.RedisEnabled = true + *operation_setting.GetMonitorSetting() = operation_setting.MonitorSetting{ + AutoPriorityDemotionEnabled: true, + StreamingFirstResultTimeoutSeconds: 1, + PriorityDeduction: 20, + } + t.Cleanup(func() { + _ = client.Close() + maxcommon.RDB, maxcommon.RedisEnabled = oldRDB, oldEnabled + *operation_setting.GetMonitorSetting() = oldSetting + }) + + baseHigh, baseLow := int64(100), int64(90) + selected, ok, err := selectChannelIdFromAbilities([]Ability{ + {ChannelId: 1, Priority: &baseHigh, Weight: 100}, + {ChannelId: 2, Priority: &baseLow, Weight: 100}, + }, 0, nil) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, 1, selected) +} + +func TestCachedRoutingCandidatesApplyRuntimeState(t *testing.T) { + setting := operation_setting.MonitorSetting{ + AutoPriorityDemotionEnabled: true, + StreamingFirstResultTimeoutSeconds: 1, + PriorityDeduction: 20, + PenaltyCooldownSeconds: 60, + RecoveryMode: operation_setting.RecoveryModeAutomatic, + } + withChannelRoutingTestState(t, setting) + _, err := channelhealth.EvaluateTimeoutGuard(context.Background(), channelhealth.AttemptEvidence{ + ChannelID: 1, + AttemptID: "cached-routing-penalty", + RequestMode: channelhealth.RequestModeStreaming, + TimeoutKind: channelhealth.TimeoutKindStreamingFirstResult, + TimeoutEligible: true, + AutoBan: true, + }, &setting) + require.NoError(t, err) + + baseHigh, baseLow := int64(100), int64(90) + candidates := buildChannelRoutingCandidates([]*Channel{ + {Id: 1, Priority: &baseHigh}, + {Id: 2, Priority: &baseLow}, + }) + require.Len(t, candidates, 2) + require.Equal(t, int64(80), candidates[0].priority) + require.Equal(t, int64(90), candidates[1].priority) +} diff --git a/model/option.go b/model/option.go index d9f0f2ed..5d53e241 100644 --- a/model/option.go +++ b/model/option.go @@ -360,6 +360,11 @@ func validateOptionUpdate(key string, value string) error { if err := validateRegisteredConfigOption(key, value); err != nil { return err } + if strings.HasPrefix(key, "monitor_setting.") { + if err := operation_setting.ValidateMonitorSettingOption(key, value); err != nil { + return err + } + } switch key { case "PreConsumedQuota": _, err := parsePreConsumedQuota(value) diff --git a/pkg/channelhealth/runtime.go b/pkg/channelhealth/runtime.go new file mode 100644 index 00000000..144a36af --- /dev/null +++ b/pkg/channelhealth/runtime.go @@ -0,0 +1,649 @@ +package channelhealth + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "strconv" + "strings" + "time" + + maxcommon "github.com/MAX-API-Next/MAX-API/common" + "github.com/MAX-API-Next/MAX-API/setting/operation_setting" + "github.com/go-redis/redis/v8" +) + +const ( + RequestModeStreaming = "streaming" + RequestModeNonStreaming = "non_streaming" + + TimeoutKindStreamingFirstResult = "streaming_first_result_timeout" + TimeoutKindNonStreamingResponse = "non_streaming_response_timeout" + TimeoutKindStreamIdle = "stream_idle_timeout" + TimeoutKindFirstResponse = "first_response" + + runtimeStateKeyPrefix = "maxapi:channel-timeout:v1:state:" + runtimeDedupeKeyPrefix = "maxapi:channel-timeout:v1:dedupe:" + runtimeStateTTL = 7 * 24 * time.Hour + runtimeDecisionTimeout = 100 * time.Millisecond + runtimeWatchAttempts = 32 +) + +var ErrRuntimeUnavailable = errors.New("channel timeout runtime is unavailable") + +type AttemptEvidence struct { + ChannelID int + AttemptID string + RequestMode string + TimeoutKind string + Success bool + TimeoutEligible bool + AutoBan bool + ObservedAt time.Time +} + +type RuntimeState struct { + ChannelID int + Penalty int64 + PenaltyUntil time.Time + RuntimeDisabled bool + DisabledUntil time.Time + ActiveSource string + Generation int64 + TimeoutCount map[string]int64 + SampleCount map[string]int64 + WindowStartedAt map[string]time.Time + LastEvent string + // LastTimeoutMode remains stable across successful samples and non-counted + // events so same-mode alert projections keep using the triggering mode. + LastTimeoutMode string + LastObservedAt time.Time + RecoveryProbeSuccessCount int64 +} + +type EvaluationResult struct { + State RuntimeState + Transition string + Deduplicated bool + Applied bool +} + +func runtimeStateKey(channelID int) string { + return fmt.Sprintf("%s%d", runtimeStateKeyPrefix, channelID) +} + +func runtimeDedupeKey(evidence AttemptEvidence) string { + hashedAttemptID := sha256.Sum256([]byte(evidence.AttemptID)) + return fmt.Sprintf("%s%d:%s:%x", runtimeDedupeKeyPrefix, evidence.ChannelID, evidence.RequestMode, hashedAttemptID) +} + +func normalizeMode(mode string) string { + if mode == RequestModeStreaming { + return RequestModeStreaming + } + return RequestModeNonStreaming +} + +func timeoutThreshold(setting *operation_setting.MonitorSetting, mode string) int { + if mode == RequestModeStreaming { + return setting.StreamingFirstResultTimeoutSeconds + } + return setting.NonStreamingResponseTimeoutSeconds +} + +func isCountableTimeout(kind string) bool { + return kind == TimeoutKindStreamingFirstResult || kind == TimeoutKindNonStreamingResponse +} + +func isSuccessfulSample(evidence AttemptEvidence) bool { + return evidence.Success && evidence.TimeoutKind == TimeoutKindFirstResponse +} + +func advancedSampleGateEnabled(setting *operation_setting.MonitorSetting) bool { + return setting != nil && (setting.TimeoutAutoDisableMinimumSamples > 0 || setting.TimeoutAutoDisableRatioPercent > 0) +} + +func countModeForSetting(setting *operation_setting.MonitorSetting, mode string) string { + if setting != nil && setting.TimeoutAutoDisableCountScope == operation_setting.TimeoutAutoDisableCountScopeCombined { + return "combined" + } + return mode +} + +func resetTimeoutWindow(state *RuntimeState) { + if state == nil { + return + } + for _, mode := range []string{RequestModeStreaming, RequestModeNonStreaming, "combined"} { + state.TimeoutCount[mode] = 0 + state.SampleCount[mode] = 0 + state.WindowStartedAt[mode] = time.Time{} + } +} + +func emptyRuntimeState(channelID int) RuntimeState { + return RuntimeState{ + ChannelID: channelID, + TimeoutCount: map[string]int64{}, + SampleCount: map[string]int64{}, + WindowStartedAt: map[string]time.Time{}, + } +} + +func decodeRuntimeState(channelID int, fields map[string]string) RuntimeState { + state := emptyRuntimeState(channelID) + state.Penalty = parseInt64(fields["penalty"]) + state.PenaltyUntil = parseTime(fields["penalty_until"]) + state.RuntimeDisabled = fields["runtime_disabled"] == "true" + state.DisabledUntil = parseTime(fields["disabled_until"]) + state.ActiveSource = fields["active_source"] + state.Generation = parseInt64(fields["generation"]) + state.LastEvent = fields["last_event"] + state.LastTimeoutMode = normalizeRuntimeTimeoutMode(fields["last_timeout_mode"]) + state.LastObservedAt = parseTime(fields["last_observed_at"]) + state.RecoveryProbeSuccessCount = parseInt64(fields["recovery_probe_success_count"]) + for _, mode := range []string{RequestModeStreaming, RequestModeNonStreaming, "combined"} { + state.TimeoutCount[mode] = parseInt64(fields["timeout_count_"+mode]) + state.SampleCount[mode] = parseInt64(fields["sample_count_"+mode]) + state.WindowStartedAt[mode] = parseTime(fields["window_start_"+mode]) + } + return state +} + +func encodeRuntimeState(state RuntimeState) map[string]interface{} { + fields := map[string]interface{}{ + "channel_id": strconv.Itoa(state.ChannelID), + "penalty": strconv.FormatInt(state.Penalty, 10), + "penalty_until": formatTime(state.PenaltyUntil), + "runtime_disabled": strconv.FormatBool(state.RuntimeDisabled), + "disabled_until": formatTime(state.DisabledUntil), + "active_source": state.ActiveSource, + "generation": strconv.FormatInt(state.Generation, 10), + "last_event": state.LastEvent, + "last_timeout_mode": state.LastTimeoutMode, + "last_observed_at": formatTime(state.LastObservedAt), + "recovery_probe_success_count": strconv.FormatInt(state.RecoveryProbeSuccessCount, 10), + } + for _, mode := range []string{RequestModeStreaming, RequestModeNonStreaming, "combined"} { + fields["timeout_count_"+mode] = strconv.FormatInt(state.TimeoutCount[mode], 10) + fields["sample_count_"+mode] = strconv.FormatInt(state.SampleCount[mode], 10) + fields["window_start_"+mode] = formatTime(state.WindowStartedAt[mode]) + } + return fields +} + +func normalizeRuntimeTimeoutMode(mode string) string { + if mode == RequestModeStreaming || mode == RequestModeNonStreaming { + return mode + } + return "" +} + +func parseInt64(value string) int64 { + parsed, _ := strconv.ParseInt(value, 10, 64) + return parsed +} + +func parseTime(value string) time.Time { + if value == "" { + return time.Time{} + } + parsed, _ := time.Parse(time.RFC3339Nano, value) + return parsed +} + +func formatTime(value time.Time) string { + if value.IsZero() { + return "" + } + return value.UTC().Format(time.RFC3339Nano) +} + +// LoadRuntimeState is read-only and fails closed for callers that need to +// apply an automatic routing mutation. +func LoadRuntimeState(ctx context.Context, channelID int) (RuntimeState, error) { + state := emptyRuntimeState(channelID) + if channelID <= 0 || !maxcommon.RedisEnabled || maxcommon.RDB == nil { + return state, ErrRuntimeUnavailable + } + fields, err := maxcommon.RDB.HGetAll(ctx, runtimeStateKey(channelID)).Result() + if err != nil && !errors.Is(err, redis.Nil) { + return state, err + } + return decodeRuntimeState(channelID, fields), nil +} + +// LoadRuntimeStates reads several channel states in one Redis round trip. A +// routing caller should treat any returned error as fail-open and keep using +// base priorities. +func LoadRuntimeStates(ctx context.Context, channelIDs []int) (map[int]RuntimeState, error) { + states := make(map[int]RuntimeState, len(channelIDs)) + if len(channelIDs) == 0 { + return states, nil + } + if !maxcommon.RedisEnabled || maxcommon.RDB == nil { + return states, ErrRuntimeUnavailable + } + if ctx == nil { + ctx = context.Background() + } + pipe := maxcommon.RDB.Pipeline() + commands := make(map[int]*redis.StringStringMapCmd, len(channelIDs)) + seen := make(map[int]struct{}, len(channelIDs)) + for _, channelID := range channelIDs { + if channelID <= 0 { + continue + } + if _, ok := seen[channelID]; ok { + continue + } + seen[channelID] = struct{}{} + commands[channelID] = pipe.HGetAll(ctx, runtimeStateKey(channelID)) + } + if _, err := pipe.Exec(ctx); err != nil { + return states, err + } + for channelID, command := range commands { + fields, err := command.Result() + if err != nil && !errors.Is(err, redis.Nil) { + return states, err + } + states[channelID] = decodeRuntimeState(channelID, fields) + } + return states, nil +} + +// ListRuntimeStates returns the bounded shared Redis projection used to +// rebuild Smart Operations after a process restart. It never reads SQL. +func ListRuntimeStates(ctx context.Context) ([]RuntimeState, error) { + if !maxcommon.RedisEnabled || maxcommon.RDB == nil { + return nil, ErrRuntimeUnavailable + } + if ctx == nil { + ctx = context.Background() + } + keys := make([]string, 0, 32) + var cursor uint64 + for { + batch, next, err := maxcommon.RDB.Scan(ctx, cursor, runtimeStateKeyPrefix+"*", 200).Result() + if err != nil { + return nil, err + } + keys = append(keys, batch...) + cursor = next + if cursor == 0 || len(keys) >= 1000 { + break + } + } + if len(keys) == 0 { + return []RuntimeState{}, nil + } + pipe := maxcommon.RDB.Pipeline() + commands := make([]*redis.StringStringMapCmd, 0, len(keys)) + for _, key := range keys { + commands = append(commands, pipe.HGetAll(ctx, key)) + } + if _, err := pipe.Exec(ctx); err != nil { + return nil, err + } + states := make([]RuntimeState, 0, len(commands)) + for index, command := range commands { + channelID, err := strconv.Atoi(strings.TrimPrefix(keys[index], runtimeStateKeyPrefix)) + if err != nil || channelID <= 0 { + continue + } + fields, err := command.Result() + if err != nil && !errors.Is(err, redis.Nil) { + return nil, err + } + states = append(states, decodeRuntimeState(channelID, fields)) + } + return states, nil +} + +// EvaluateTimeoutGuard performs one idempotent runtime state transition. It +// never updates SQL channel state and returns ErrRuntimeUnavailable when a +// shared Redis decision cannot be made. +func EvaluateTimeoutGuard(ctx context.Context, evidence AttemptEvidence, setting *operation_setting.MonitorSetting) (EvaluationResult, error) { + result := EvaluationResult{State: emptyRuntimeState(evidence.ChannelID)} + if evidence.ChannelID <= 0 || evidence.AttemptID == "" || setting == nil { + return result, errors.New("invalid channel timeout evidence") + } + if !maxcommon.RedisEnabled || maxcommon.RDB == nil { + return result, ErrRuntimeUnavailable + } + if !setting.AutoPriorityDemotionEnabled && !setting.TimeoutAutoDisableEnabled { + return result, nil + } + if evidence.Success && (!evidence.AutoBan || !setting.TimeoutAutoDisableEnabled || !advancedSampleGateEnabled(setting)) { + return result, nil + } + decisionCtx, cancel := boundedRuntimeContext(ctx) + defer cancel() + ctx = decisionCtx + now := evidence.ObservedAt + if now.IsZero() { + now = time.Now() + } + mode := normalizeMode(evidence.RequestMode) + key := runtimeStateKey(evidence.ChannelID) + dedupeKey := runtimeDedupeKey(evidence) + windowTTL := time.Duration(setting.TimeoutAutoDisableWindowSeconds) * time.Second + if windowTTL <= 0 { + windowTTL = time.Hour + } + + err := watchRuntimeState(ctx, []string{key, dedupeKey}, func(tx *redis.Tx) error { + // WATCH may invoke the callback more than once. Never expose a result + // assembled by a transaction that is later retried or rejected. + result = EvaluationResult{State: emptyRuntimeState(evidence.ChannelID)} + deduped, err := tx.Exists(ctx, dedupeKey).Result() + if err != nil { + return err + } + fields, err := tx.HGetAll(ctx, key).Result() + if err != nil && !errors.Is(err, redis.Nil) { + return err + } + state := decodeRuntimeState(evidence.ChannelID, fields) + result.State = state + if deduped > 0 { + result.Deduplicated = true + return nil + } + + shouldCount := evidence.TimeoutEligible && isCountableTimeout(evidence.TimeoutKind) && + setting.TimeoutAutoDisableEnabled && evidence.AutoBan && + timeoutThreshold(setting, mode) > 0 + shouldRecordSample := setting.TimeoutAutoDisableEnabled && evidence.AutoBan && + advancedSampleGateEnabled(setting) && timeoutThreshold(setting, mode) > 0 && + (isSuccessfulSample(evidence) || shouldCount) + countMode := countModeForSetting(setting, mode) + if shouldCount || shouldRecordSample { + windowStart := state.WindowStartedAt[countMode] + if windowStart.IsZero() || now.Sub(windowStart) >= windowTTL { + windowStart = now + state.TimeoutCount[countMode] = 0 + state.SampleCount[countMode] = 0 + } + state.WindowStartedAt[countMode] = windowStart + } + if shouldCount { + state.TimeoutCount[countMode]++ + state.LastTimeoutMode = mode + } + if shouldRecordSample { + state.SampleCount[countMode]++ + } + + if evidence.TimeoutEligible && isCountableTimeout(evidence.TimeoutKind) && + setting.AutoPriorityDemotionEnabled && + setting.PriorityDeduction > 0 && + timeoutThreshold(setting, mode) > 0 && + !state.RuntimeDisabled { + previouslyActive := state.Penalty > 0 && (state.PenaltyUntil.IsZero() || now.Before(state.PenaltyUntil)) + state.Penalty = int64(setting.PriorityDeduction) + if setting.RecoveryMode == operation_setting.RecoveryModeAutomatic { + until := now.Add(time.Duration(setting.PenaltyCooldownSeconds) * time.Second) + if setting.ExtendOnRepeatTimeout && previouslyActive && until.Before(state.PenaltyUntil) { + until = state.PenaltyUntil + } + state.PenaltyUntil = until + } else { + state.PenaltyUntil = time.Time{} + } + if !previouslyActive { + result.Transition = "priority_demotion_firing" + } else { + result.Transition = "priority_demotion_updated" + } + result.Applied = true + } + + if (shouldCount || shouldRecordSample) && setting.TimeoutAutoDisableCount > 0 { + if state.TimeoutCount[countMode] >= int64(setting.TimeoutAutoDisableCount) && + (!advancedSampleGateEnabled(setting) || + (state.SampleCount[countMode] >= int64(setting.TimeoutAutoDisableMinimumSamples) && + (setting.TimeoutAutoDisableRatioPercent <= 0 || + float64(state.TimeoutCount[countMode])*100 >= setting.TimeoutAutoDisableRatioPercent*float64(state.SampleCount[countMode])))) && + !state.RuntimeDisabled { + state.RuntimeDisabled = true + state.ActiveSource = "repeated_timeout" + if setting.TimeoutAutoDisableDurationSeconds > 0 { + state.DisabledUntil = now.Add(time.Duration(setting.TimeoutAutoDisableDurationSeconds) * time.Second) + } + state.Penalty = 0 + state.PenaltyUntil = time.Time{} + result.Transition = "timeout_auto_disabled" + result.Applied = true + } + } + + state.Generation++ + state.LastEvent = evidence.TimeoutKind + state.LastObservedAt = now + result.State = state + pipe := tx.TxPipeline() + pipe.HSet(ctx, key, encodeRuntimeState(state)) + pipe.Set(ctx, dedupeKey, "1", windowTTL) + if state.RuntimeDisabled && + (setting.TimeoutAutoDisableDurationSeconds == 0 || + setting.TimeoutAutoDisableRecoveryMode == operation_setting.TimeoutAutoDisableRecoveryManual) || + state.Penalty > 0 && setting.RecoveryMode == operation_setting.RecoveryModeManual { + pipe.Persist(ctx, key) + } else { + pipe.Expire(ctx, key, runtimeStateTTL) + } + _, err = pipe.Exec(ctx) + return err + }) + if err != nil { + if errors.Is(err, redis.TxFailedErr) { + return EvaluationResult{State: emptyRuntimeState(evidence.ChannelID)}, fmt.Errorf("channel timeout runtime contention: %w", err) + } + return EvaluationResult{State: emptyRuntimeState(evidence.ChannelID)}, err + } + return result, nil +} + +func boundedRuntimeContext(ctx context.Context) (context.Context, context.CancelFunc) { + if ctx == nil { + ctx = context.Background() + } + return context.WithTimeout(ctx, runtimeDecisionTimeout) +} + +// watchRuntimeState retries optimistic Redis transactions that lost a +// concurrent update. The retry budget is bounded by the caller's decision +// context so contention never turns into an unbounded relay delay. +func watchRuntimeState(ctx context.Context, keys []string, fn func(*redis.Tx) error) error { + var err error + for attempt := 0; attempt < runtimeWatchAttempts; attempt++ { + err = maxcommon.RDB.Watch(ctx, fn, keys...) + if !errors.Is(err, redis.TxFailedErr) { + return err + } + if ctx.Err() != nil { + return ctx.Err() + } + } + return err +} + +// RecoverRuntimeState records one controlled recovery probe. Probe attempts +// are separate from production timeout evidence and can only clear a runtime +// disablement after the configured number of consecutive successes. +func RecoverRuntimeState(ctx context.Context, channelID int, probeID string, success bool, setting *operation_setting.MonitorSetting) (EvaluationResult, error) { + result := EvaluationResult{State: emptyRuntimeState(channelID)} + if channelID <= 0 || strings.TrimSpace(probeID) == "" || setting == nil { + return result, errors.New("invalid channel recovery probe") + } + if !maxcommon.RedisEnabled || maxcommon.RDB == nil { + return result, ErrRuntimeUnavailable + } + if setting.TimeoutAutoDisableRecoveryMode != operation_setting.TimeoutAutoDisableRecoveryProbe || + setting.TimeoutAutoDisableDurationSeconds <= 0 { + return result, errors.New("automatic channel recovery is not enabled") + } + decisionCtx, cancel := boundedRuntimeContext(ctx) + defer cancel() + key := runtimeStateKey(channelID) + dedupeKey := runtimeRecoveryDedupeKey(channelID, probeID) + now := time.Now() + probeThreshold := setting.TimeoutAutoDisableSuccessfulProbeCount + if probeThreshold <= 0 { + probeThreshold = 1 + } + probeTTL := time.Duration(setting.TimeoutAutoDisableWindowSeconds) * time.Second + if probeTTL <= 0 { + probeTTL = time.Hour + } + + err := watchRuntimeState(decisionCtx, []string{key, dedupeKey}, func(tx *redis.Tx) error { + result = EvaluationResult{State: emptyRuntimeState(channelID)} + deduped, err := tx.Exists(decisionCtx, dedupeKey).Result() + if err != nil { + return err + } + fields, err := tx.HGetAll(decisionCtx, key).Result() + if err != nil && !errors.Is(err, redis.Nil) { + return err + } + state := decodeRuntimeState(channelID, fields) + result.State = state + if !state.RuntimeDisabled { + return nil + } + if deduped > 0 { + result.Deduplicated = true + return nil + } + if !state.DisabledUntil.IsZero() && now.Before(state.DisabledUntil) { + return nil + } + if success { + state.RecoveryProbeSuccessCount++ + if state.RecoveryProbeSuccessCount >= int64(probeThreshold) { + resetTimeoutWindow(&state) + state.RuntimeDisabled = false + state.DisabledUntil = time.Time{} + state.ActiveSource = "" + state.Penalty = 0 + state.PenaltyUntil = time.Time{} + state.RecoveryProbeSuccessCount = 0 + state.LastEvent = "recovery_probe_succeeded" + result.Transition = "runtime_recovered" + result.Applied = true + } else { + state.LastEvent = "recovery_probe_succeeded" + } + } else { + state.RecoveryProbeSuccessCount = 0 + state.LastEvent = "recovery_probe_failed" + } + state.Generation++ + state.LastObservedAt = now + result.State = state + pipe := tx.TxPipeline() + pipe.HSet(decisionCtx, key, encodeRuntimeState(state)) + pipe.Set(decisionCtx, dedupeKey, "1", probeTTL) + pipe.Expire(decisionCtx, key, runtimeStateTTL) + _, err = pipe.Exec(decisionCtx) + return err + }) + if err != nil { + if errors.Is(err, redis.TxFailedErr) { + return EvaluationResult{State: emptyRuntimeState(channelID)}, fmt.Errorf("channel recovery runtime contention: %w", err) + } + return EvaluationResult{State: emptyRuntimeState(channelID)}, err + } + return result, nil +} + +// RestoreRuntimeState clears only the timeout strategy's Redis runtime state. +// It never changes the persisted channel status or ability rows. +func RestoreRuntimeState(ctx context.Context, channelID int) (EvaluationResult, error) { + result := EvaluationResult{State: emptyRuntimeState(channelID)} + if channelID <= 0 || !maxcommon.RedisEnabled || maxcommon.RDB == nil { + return result, ErrRuntimeUnavailable + } + decisionCtx, cancel := boundedRuntimeContext(ctx) + defer cancel() + key := runtimeStateKey(channelID) + err := watchRuntimeState(decisionCtx, []string{key}, func(tx *redis.Tx) error { + result = EvaluationResult{State: emptyRuntimeState(channelID)} + fields, err := tx.HGetAll(decisionCtx, key).Result() + if err != nil && !errors.Is(err, redis.Nil) { + return err + } + state := decodeRuntimeState(channelID, fields) + if !state.RuntimeDisabled && state.Penalty <= 0 { + result.State = state + return nil + } + state.RuntimeDisabled = false + state.DisabledUntil = time.Time{} + state.ActiveSource = "" + state.Penalty = 0 + state.PenaltyUntil = time.Time{} + state.RecoveryProbeSuccessCount = 0 + resetTimeoutWindow(&state) + state.Generation++ + state.LastEvent = "manual_recovery" + state.LastObservedAt = time.Now() + result.State = state + result.Transition = "runtime_recovered" + result.Applied = true + pipe := tx.TxPipeline() + pipe.HSet(decisionCtx, key, encodeRuntimeState(state)) + pipe.Expire(decisionCtx, key, runtimeStateTTL) + _, err = pipe.Exec(decisionCtx) + return err + }) + if err != nil { + if errors.Is(err, redis.TxFailedErr) { + return EvaluationResult{State: emptyRuntimeState(channelID)}, fmt.Errorf("manual channel recovery contention: %w", err) + } + return EvaluationResult{State: emptyRuntimeState(channelID)}, err + } + return result, nil +} + +func EffectivePriority(base int64, state RuntimeState, now time.Time) int64 { + if now.IsZero() { + now = time.Now() + } + if state.RuntimeDisabled { + return base + } + if state.Penalty <= 0 || (!state.PenaltyUntil.IsZero() && !now.Before(state.PenaltyUntil)) { + return base + } + effective := base - state.Penalty + if effective < 0 { + return 0 + } + return effective +} + +func RuntimeStateKey(channelID int) string { + return runtimeStateKey(channelID) +} + +func runtimeRecoveryDedupeKey(channelID int, probeID string) string { + hashedProbeID := sha256.Sum256([]byte(strings.TrimSpace(probeID))) + return fmt.Sprintf("%srecovery:%d:%x", runtimeDedupeKeyPrefix, channelID, hashedProbeID) +} + +func RuntimeStateSummary(state RuntimeState) string { + parts := []string{ + "channel=" + strconv.Itoa(state.ChannelID), + "penalty=" + strconv.FormatInt(state.Penalty, 10), + "disabled=" + strconv.FormatBool(state.RuntimeDisabled), + } + if state.ActiveSource != "" { + parts = append(parts, "source="+state.ActiveSource) + } + return strings.Join(parts, " ") +} diff --git a/pkg/channelhealth/runtime_test.go b/pkg/channelhealth/runtime_test.go new file mode 100644 index 00000000..9adbac9f --- /dev/null +++ b/pkg/channelhealth/runtime_test.go @@ -0,0 +1,411 @@ +package channelhealth + +import ( + "context" + "strconv" + "strings" + "sync" + "testing" + "time" + + maxcommon "github.com/MAX-API-Next/MAX-API/common" + "github.com/MAX-API-Next/MAX-API/setting/operation_setting" + "github.com/alicebob/miniredis/v2" + "github.com/go-redis/redis/v8" + "github.com/stretchr/testify/require" +) + +func testRuntimeSetting() *operation_setting.MonitorSetting { + return &operation_setting.MonitorSetting{ + AutoPriorityDemotionEnabled: true, + StreamingFirstResultTimeoutSeconds: 10, + PriorityDeduction: 10, + PenaltyCooldownSeconds: 60, + RecoveryMode: operation_setting.RecoveryModeAutomatic, + ExtendOnRepeatTimeout: true, + TimeoutAutoDisableEnabled: true, + TimeoutAutoDisableCount: 2, + TimeoutAutoDisableWindowSeconds: 3600, + TimeoutAutoDisableCountScope: operation_setting.TimeoutAutoDisableCountScopeSameMode, + TimeoutAutoDisableDurationSeconds: 1800, + TimeoutAutoDisableRecoveryMode: operation_setting.TimeoutAutoDisableRecoveryProbe, + } +} + +func withRuntimeRedis(t *testing.T) { + t.Helper() + server := miniredis.RunT(t) + oldRDB, oldEnabled := maxcommon.RDB, maxcommon.RedisEnabled + maxcommon.RDB = redis.NewClient(&redis.Options{Addr: server.Addr()}) + maxcommon.RedisEnabled = true + t.Cleanup(func() { + _ = maxcommon.RDB.Close() + maxcommon.RDB, maxcommon.RedisEnabled = oldRDB, oldEnabled + }) +} + +func TestEvaluateTimeoutGuardAppliesPenaltyAndDeduplicates(t *testing.T) { + withRuntimeRedis(t) + setting := testRuntimeSetting() + now := time.Now() + + first, err := EvaluateTimeoutGuard(context.Background(), AttemptEvidence{ + ChannelID: 936, + AttemptID: "attempt-1", + RequestMode: RequestModeStreaming, + TimeoutKind: TimeoutKindStreamingFirstResult, + TimeoutEligible: true, + AutoBan: true, + ObservedAt: now, + }, setting) + require.NoError(t, err) + require.True(t, first.Applied) + require.Equal(t, "priority_demotion_firing", first.Transition) + require.Equal(t, int64(10), first.State.Penalty) + require.Equal(t, int64(1), first.State.TimeoutCount[RequestModeStreaming]) + + duplicate, err := EvaluateTimeoutGuard(context.Background(), AttemptEvidence{ + ChannelID: 936, + AttemptID: "attempt-1", + RequestMode: RequestModeStreaming, + TimeoutKind: TimeoutKindStreamingFirstResult, + TimeoutEligible: true, + AutoBan: true, + ObservedAt: now.Add(time.Second), + }, setting) + require.NoError(t, err) + require.True(t, duplicate.Deduplicated) + require.Equal(t, int64(1), duplicate.State.TimeoutCount[RequestModeStreaming]) +} + +func TestEvaluateTimeoutGuardRetriesConcurrentRedisTransactions(t *testing.T) { + withRuntimeRedis(t) + setting := testRuntimeSetting() + setting.TimeoutAutoDisableCount = 100 + const attempts = 8 + errorsCh := make(chan error, attempts) + var waitGroup sync.WaitGroup + waitGroup.Add(attempts) + for index := 0; index < attempts; index++ { + index := index + go func() { + defer waitGroup.Done() + _, err := EvaluateTimeoutGuard(context.Background(), AttemptEvidence{ + ChannelID: 936, + AttemptID: "concurrent-attempt-" + strconv.Itoa(index), + RequestMode: RequestModeStreaming, + TimeoutKind: TimeoutKindStreamingFirstResult, + TimeoutEligible: true, + AutoBan: true, + ObservedAt: time.Now(), + }, setting) + errorsCh <- err + }() + } + waitGroup.Wait() + close(errorsCh) + for err := range errorsCh { + require.NoError(t, err) + } + + state, err := LoadRuntimeState(context.Background(), 936) + require.NoError(t, err) + require.Equal(t, int64(attempts), state.TimeoutCount[RequestModeStreaming]) +} + +func TestEvaluateTimeoutGuardBoundsUnavailableRedis(t *testing.T) { + oldRDB, oldEnabled := maxcommon.RDB, maxcommon.RedisEnabled + client := redis.NewClient(&redis.Options{ + Addr: "127.0.0.1:1", + MaxRetries: 0, + DialTimeout: 20 * time.Millisecond, + ReadTimeout: 20 * time.Millisecond, + WriteTimeout: 20 * time.Millisecond, + }) + maxcommon.RDB = client + maxcommon.RedisEnabled = true + t.Cleanup(func() { + _ = client.Close() + maxcommon.RDB, maxcommon.RedisEnabled = oldRDB, oldEnabled + }) + + started := time.Now() + _, err := EvaluateTimeoutGuard(context.Background(), AttemptEvidence{ + ChannelID: 936, + AttemptID: "unavailable-redis", + RequestMode: RequestModeStreaming, + TimeoutKind: TimeoutKindStreamingFirstResult, + TimeoutEligible: true, + AutoBan: true, + }, testRuntimeSetting()) + require.Error(t, err) + require.Less(t, time.Since(started), 500*time.Millisecond) +} + +func TestEvaluateTimeoutGuardAutoDisablesAfterConfiguredCount(t *testing.T) { + withRuntimeRedis(t) + setting := testRuntimeSetting() + now := time.Now() + for _, id := range []string{"attempt-1", "attempt-2"} { + result, err := EvaluateTimeoutGuard(context.Background(), AttemptEvidence{ + ChannelID: 936, + AttemptID: id, + RequestMode: RequestModeStreaming, + TimeoutKind: TimeoutKindStreamingFirstResult, + TimeoutEligible: true, + AutoBan: true, + ObservedAt: now, + }, setting) + require.NoError(t, err) + if id == "attempt-2" { + require.Equal(t, "timeout_auto_disabled", result.Transition) + require.True(t, result.State.RuntimeDisabled) + require.WithinDuration(t, now.Add(30*time.Minute), result.State.DisabledUntil, time.Second) + require.Zero(t, result.State.Penalty) + } + } +} + +func TestEvaluateTimeoutGuardAdvancedGateUsesSuccessfulSamplesAndRatio(t *testing.T) { + withRuntimeRedis(t) + setting := testRuntimeSetting() + setting.TimeoutAutoDisableMinimumSamples = 3 + setting.TimeoutAutoDisableRatioPercent = 50 + now := time.Now() + for _, id := range []string{"success-1", "success-2", "success-3", "timeout-1", "timeout-2"} { + evidence := AttemptEvidence{ + ChannelID: 936, + AttemptID: id, + RequestMode: RequestModeStreaming, + AutoBan: true, + ObservedAt: now, + } + if strings.HasPrefix(id, "success") { + evidence.Success = true + evidence.TimeoutKind = TimeoutKindFirstResponse + } else { + evidence.TimeoutEligible = true + evidence.TimeoutKind = TimeoutKindStreamingFirstResult + } + result, err := EvaluateTimeoutGuard(context.Background(), evidence, setting) + require.NoError(t, err) + if id == "timeout-2" { + require.False(t, result.State.RuntimeDisabled) + require.Equal(t, int64(2), result.State.TimeoutCount[RequestModeStreaming]) + require.Equal(t, int64(5), result.State.SampleCount[RequestModeStreaming]) + } + } + + result, err := EvaluateTimeoutGuard(context.Background(), AttemptEvidence{ + ChannelID: 936, + AttemptID: "timeout-3", + RequestMode: RequestModeStreaming, + TimeoutKind: TimeoutKindStreamingFirstResult, + TimeoutEligible: true, + AutoBan: true, + ObservedAt: now, + }, setting) + require.NoError(t, err) + require.True(t, result.State.RuntimeDisabled) + require.Equal(t, "timeout_auto_disabled", result.Transition) + require.Equal(t, int64(3), result.State.TimeoutCount[RequestModeStreaming]) + require.Equal(t, int64(6), result.State.SampleCount[RequestModeStreaming]) +} + +func TestEvaluateTimeoutGuardSuccessfulSampleIsNoOpWhenAdvancedGateDisabled(t *testing.T) { + withRuntimeRedis(t) + setting := testRuntimeSetting() + setting.TimeoutAutoDisableRatioPercent = 0 + result, err := EvaluateTimeoutGuard(context.Background(), AttemptEvidence{ + ChannelID: 936, + AttemptID: "success-without-gate", + RequestMode: RequestModeStreaming, + TimeoutKind: TimeoutKindFirstResponse, + Success: true, + AutoBan: true, + }, setting) + require.NoError(t, err) + require.False(t, result.Applied) + keys, err := maxcommon.RDB.Keys(context.Background(), "maxapi:channel-timeout:v1:*").Result() + require.NoError(t, err) + require.Empty(t, keys) +} + +func TestEvaluateTimeoutGuardPreservesLastTimeoutModeAcrossSuccessfulSamples(t *testing.T) { + withRuntimeRedis(t) + setting := testRuntimeSetting() + setting.TimeoutAutoDisableMinimumSamples = 1 + setting.TimeoutAutoDisableRatioPercent = 50 + now := time.Now() + + _, err := EvaluateTimeoutGuard(context.Background(), AttemptEvidence{ + ChannelID: 936, + AttemptID: "streaming-timeout", + RequestMode: RequestModeStreaming, + TimeoutKind: TimeoutKindStreamingFirstResult, + TimeoutEligible: true, + AutoBan: true, + ObservedAt: now, + }, setting) + require.NoError(t, err) + + result, err := EvaluateTimeoutGuard(context.Background(), AttemptEvidence{ + ChannelID: 936, + AttemptID: "streaming-success", + RequestMode: RequestModeStreaming, + TimeoutKind: TimeoutKindFirstResponse, + Success: true, + AutoBan: true, + ObservedAt: now.Add(time.Second), + }, setting) + require.NoError(t, err) + require.Equal(t, RequestModeStreaming, result.State.LastTimeoutMode) + require.Equal(t, TimeoutKindFirstResponse, result.State.LastEvent) +} + +func TestEvaluateTimeoutGuardIsNoOpWhenPoliciesAreDisabled(t *testing.T) { + withRuntimeRedis(t) + setting := &operation_setting.MonitorSetting{} + result, err := EvaluateTimeoutGuard(context.Background(), AttemptEvidence{ + ChannelID: 936, + AttemptID: "disabled-policy-attempt", + RequestMode: RequestModeStreaming, + TimeoutKind: TimeoutKindStreamingFirstResult, + TimeoutEligible: true, + AutoBan: true, + }, setting) + require.NoError(t, err) + require.False(t, result.Applied) + require.False(t, result.Deduplicated) + keys, err := maxcommon.RDB.Keys(context.Background(), "maxapi:channel-timeout:v1:*").Result() + require.NoError(t, err) + require.Empty(t, keys) +} + +func TestEffectivePriorityExpiresAndClampsAtZero(t *testing.T) { + state := RuntimeState{Penalty: 20, PenaltyUntil: time.Now().Add(time.Minute)} + require.Equal(t, int64(80), EffectivePriority(100, state, time.Now())) + require.Equal(t, int64(100), EffectivePriority(100, state, time.Now().Add(2*time.Minute))) + require.Equal(t, int64(0), EffectivePriority(10, state, time.Now())) +} + +func TestStreamIdleTimeoutDoesNotCountOrDemote(t *testing.T) { + withRuntimeRedis(t) + setting := testRuntimeSetting() + result, err := EvaluateTimeoutGuard(context.Background(), AttemptEvidence{ + ChannelID: 936, + AttemptID: "attempt-idle", + RequestMode: RequestModeStreaming, + TimeoutKind: TimeoutKindStreamIdle, + TimeoutEligible: true, + AutoBan: true, + }, setting) + require.NoError(t, err) + require.False(t, result.Applied) + require.Zero(t, result.State.Penalty) + require.Zero(t, result.State.TimeoutCount[RequestModeStreaming]) +} + +func TestListRuntimeStatesRebuildsTheSmartOpsProjection(t *testing.T) { + withRuntimeRedis(t) + setting := testRuntimeSetting() + now := time.Now() + _, err := EvaluateTimeoutGuard(context.Background(), AttemptEvidence{ + ChannelID: 936, + AttemptID: "restart-visible-attempt", + RequestMode: RequestModeStreaming, + TimeoutKind: TimeoutKindStreamingFirstResult, + TimeoutEligible: true, + AutoBan: true, + ObservedAt: now, + }, setting) + require.NoError(t, err) + + states, err := ListRuntimeStates(context.Background()) + require.NoError(t, err) + require.Len(t, states, 1) + require.Equal(t, 936, states[0].ChannelID) + require.Equal(t, int64(10), states[0].Penalty) + require.Equal(t, int64(1), states[0].TimeoutCount[RequestModeStreaming]) +} + +func TestRecoverRuntimeStateRequiresConfiguredSuccessfulProbes(t *testing.T) { + withRuntimeRedis(t) + setting := testRuntimeSetting() + setting.TimeoutAutoDisableSuccessfulProbeCount = 2 + now := time.Now().Add(-time.Minute) + key := RuntimeStateKey(936) + _, err := maxcommon.RDB.HSet(context.Background(), key, map[string]interface{}{ + "channel_id": "936", + "runtime_disabled": "true", + "disabled_until": now.Format(time.RFC3339Nano), + "active_source": "repeated_timeout", + "generation": "1", + "timeout_count_streaming": "2", + "sample_count_streaming": "4", + "window_start_streaming": now.Add(-time.Minute).Format(time.RFC3339Nano), + }).Result() + require.NoError(t, err) + + first, err := RecoverRuntimeState(context.Background(), 936, "probe-1", true, setting) + require.NoError(t, err) + require.False(t, first.Applied) + require.True(t, first.State.RuntimeDisabled) + require.Equal(t, int64(1), first.State.RecoveryProbeSuccessCount) + + second, err := RecoverRuntimeState(context.Background(), 936, "probe-2", true, setting) + require.NoError(t, err) + require.True(t, second.Applied) + require.Equal(t, "runtime_recovered", second.Transition) + require.False(t, second.State.RuntimeDisabled) + require.Zero(t, second.State.RecoveryProbeSuccessCount) + require.Zero(t, second.State.TimeoutCount[RequestModeStreaming]) + require.Zero(t, second.State.SampleCount[RequestModeStreaming]) + require.True(t, second.State.WindowStartedAt[RequestModeStreaming].IsZero()) +} + +func TestRecoverRuntimeStateFailedProbeKeepsChannelDisabled(t *testing.T) { + withRuntimeRedis(t) + setting := testRuntimeSetting() + now := time.Now().Add(-time.Minute) + key := RuntimeStateKey(936) + _, err := maxcommon.RDB.HSet(context.Background(), key, map[string]interface{}{ + "channel_id": "936", + "runtime_disabled": "true", + "disabled_until": now.Format(time.RFC3339Nano), + "active_source": "repeated_timeout", + }).Result() + require.NoError(t, err) + + result, err := RecoverRuntimeState(context.Background(), 936, "probe-failed", false, setting) + require.NoError(t, err) + require.False(t, result.Applied) + require.True(t, result.State.RuntimeDisabled) + require.Zero(t, result.State.RecoveryProbeSuccessCount) + require.Equal(t, "recovery_probe_failed", result.State.LastEvent) +} + +func TestRestoreRuntimeStateDoesNotTouchDatabaseState(t *testing.T) { + withRuntimeRedis(t) + key := RuntimeStateKey(936) + _, err := maxcommon.RDB.HSet(context.Background(), key, map[string]interface{}{ + "channel_id": "936", + "runtime_disabled": "true", + "active_source": "repeated_timeout", + "penalty": "10", + "timeout_count_streaming": "2", + "sample_count_streaming": "4", + "window_start_streaming": time.Now().Add(-time.Minute).Format(time.RFC3339Nano), + }).Result() + require.NoError(t, err) + + result, err := RestoreRuntimeState(context.Background(), 936) + require.NoError(t, err) + require.True(t, result.Applied) + require.Equal(t, "runtime_recovered", result.Transition) + require.False(t, result.State.RuntimeDisabled) + require.Zero(t, result.State.Penalty) + require.Zero(t, result.State.TimeoutCount[RequestModeStreaming]) + require.Zero(t, result.State.SampleCount[RequestModeStreaming]) + require.True(t, result.State.WindowStartedAt[RequestModeStreaming].IsZero()) +} diff --git a/relay/audio_handler.go b/relay/audio_handler.go index 0772dfd7..42d2a050 100644 --- a/relay/audio_handler.go +++ b/relay/audio_handler.go @@ -67,6 +67,9 @@ func AudioHelper(c *gin.Context, info *relaycommon.RelayInfo) (maxAPIError *type service.ResetStatusCode(maxAPIError, statusCodeMappingStr) return maxAPIError } + if !info.IsStream { + info.SetFirstResponseTime() + } if usage.(*dto.Usage).CompletionTokenDetails.AudioTokens > 0 || usage.(*dto.Usage).PromptTokensDetails.AudioTokens > 0 { service.PostAudioConsumeQuota(c, info, usage.(*dto.Usage), "") } else { diff --git a/relay/channel/api_request.go b/relay/channel/api_request.go index 2ce9b5b8..4fe90f9c 100644 --- a/relay/channel/api_request.go +++ b/relay/channel/api_request.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "net" "net/http" "net/http/httptrace" "regexp" @@ -516,6 +517,7 @@ func doRequest(c *gin.Context, req *http.Request, info *common.RelayInfo) (*http } else { client = service.GetHttpClient() } + client = clientWithNonStreamingTimeout(client, info) var stopPinger context.CancelFunc var pingerDone <-chan struct{} @@ -539,12 +541,21 @@ func doRequest(c *gin.Context, req *http.Request, info *common.RelayInfo) (*http resp, err := client.Do(req) if err != nil { + if isUpstreamTimeout(err) && info != nil { + info.RecordChannelTimeout("transport_timeout") + } logger.LogError(c, "do request failed: "+err.Error()) return nil, types.NewError(err, types.ErrorCodeDoRequestFailed, types.ErrOptionWithHideErrMsg("upstream error: do request failed")) } if resp == nil { return nil, errors.New("resp is nil") } + if info != nil && !info.IsStream && resp.Body != nil { + setting := operation_setting.GetMonitorSetting() + if setting != nil && setting.NonStreamingResponseTimeoutSeconds > 0 { + resp.Body = &timeoutTrackingBody{ReadCloser: resp.Body, info: info} + } + } if upID := resp.Header.Get(common2.RequestIdKey); upID != "" { c.Set(common2.UpstreamRequestIdKey, upID) @@ -555,6 +566,47 @@ func doRequest(c *gin.Context, req *http.Request, info *common.RelayInfo) (*http return resp, nil } +func clientWithNonStreamingTimeout(client *http.Client, info *common.RelayInfo) *http.Client { + if client == nil || info == nil || info.IsStream { + return client + } + setting := operation_setting.GetMonitorSetting() + if setting == nil || setting.NonStreamingResponseTimeoutSeconds <= 0 { + return client + } + timeout := time.Duration(setting.NonStreamingResponseTimeoutSeconds) * time.Second + if client.Timeout > 0 && client.Timeout <= timeout { + return client + } + clone := *client + clone.Timeout = timeout + return &clone +} + +type timeoutTrackingBody struct { + io.ReadCloser + info *common.RelayInfo +} + +func (body *timeoutTrackingBody) Read(p []byte) (int, error) { + n, err := body.ReadCloser.Read(p) + if err != nil && isUpstreamTimeout(err) && body.info != nil { + body.info.RecordChannelTimeout("request_timeout") + } + return n, err +} + +func isUpstreamTimeout(err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.DeadlineExceeded) { + return true + } + var netErr net.Error + return errors.As(err, &netErr) && netErr.Timeout() +} + func newTaskHTTPRequest(method string, fullRequestURL string, requestBody io.Reader, info *common.RelayInfo) (*http.Request, error) { req, err := http.NewRequest(method, fullRequestURL, requestBody) if err != nil { diff --git a/relay/channel/aws/relay-aws.go b/relay/channel/aws/relay-aws.go index 1c7de080..aac99b6b 100644 --- a/relay/channel/aws/relay-aws.go +++ b/relay/channel/aws/relay-aws.go @@ -278,11 +278,11 @@ func awsStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, a *Adaptor) ( for event := range stream.Events() { switch v := event.(type) { case *bedrockruntimeTypes.ResponseStreamMemberChunk: - info.SetFirstResponseTime() respErr := claude.HandleStreamResponseData(c, info, claudeInfo, string(v.Value.Bytes)) if respErr != nil { return respErr, nil } + info.SetFirstResponseTime() case *bedrockruntimeTypes.UnknownUnionMember: fmt.Println("unknown tag:", v.Tag) return types.NewError(errors.New("unknown response type"), types.ErrorCodeInvalidRequest), nil diff --git a/relay/channel/cloudflare/first_response_test.go b/relay/channel/cloudflare/first_response_test.go new file mode 100644 index 00000000..9043a759 --- /dev/null +++ b/relay/channel/cloudflare/first_response_test.go @@ -0,0 +1,94 @@ +package cloudflare + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + relaycommon "github.com/MAX-API-Next/MAX-API/relay/common" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +func newCloudflareResponseContext() (*gin.Context, *httptest.ResponseRecorder) { + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/", nil) + return c, recorder +} + +func newCloudflareResponseInfo() *relaycommon.RelayInfo { + return &relaycommon.RelayInfo{ + ChannelMeta: &relaycommon.ChannelMeta{ + UpstreamModelName: "cloudflare-test", + }, + } +} + +func TestCloudflareStreamHandlerRecordsAfterValidPayload(t *testing.T) { + gin.SetMode(gin.TestMode) + c, recorder := newCloudflareResponseContext() + info := newCloudflareResponseInfo() + body := "data: not-json\n" + `data: {"choices":[]}` + "\n" + `data: {"choices":[{"delta":{"content":"hello"}}]}` + "\n" + "data: [DONE]\n" + + apiErr, usage := cfStreamHandler(c, info, &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(body)), + }) + + require.Nil(t, apiErr) + require.NotNil(t, usage) + require.True(t, info.HasRecordedChannelFirstResponse()) + require.Contains(t, recorder.Body.String(), `"content":"hello"`) +} + +func TestCloudflareNonStreamingHandlersRecordAfterSuccessfulWrite(t *testing.T) { + gin.SetMode(gin.TestMode) + + t.Run("chat", func(t *testing.T) { + c, _ := newCloudflareResponseContext() + info := newCloudflareResponseInfo() + body := `{"choices":[{"message":{"role":"assistant","content":"hello"}}]}` + + apiErr, usage := cfHandler(c, info, &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(body)), + }) + + require.Nil(t, apiErr) + require.NotNil(t, usage) + require.True(t, info.HasRecordedChannelFirstResponse()) + }) + + t.Run("chat-empty-choices", func(t *testing.T) { + c, _ := newCloudflareResponseContext() + info := newCloudflareResponseInfo() + body := `{"choices":[]}` + + apiErr, usage := cfHandler(c, info, &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(body)), + }) + + require.Nil(t, apiErr) + require.NotNil(t, usage) + require.False(t, info.HasRecordedChannelFirstResponse()) + }) + + t.Run("stt", func(t *testing.T) { + c, _ := newCloudflareResponseContext() + info := newCloudflareResponseInfo() + body := `{"result":{"text":"hello"}}` + + apiErr, usage := cfSTTHandler(c, info, &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(body)), + }) + + require.Nil(t, apiErr) + require.NotNil(t, usage) + require.True(t, info.HasRecordedChannelFirstResponse()) + }) +} diff --git a/relay/channel/cloudflare/relay_cloudflare.go b/relay/channel/cloudflare/relay_cloudflare.go index 9682dd56..683a8f50 100644 --- a/relay/channel/cloudflare/relay_cloudflare.go +++ b/relay/channel/cloudflare/relay_cloudflare.go @@ -6,7 +6,6 @@ import ( "io" "net/http" "strings" - "time" "github.com/MAX-API-Next/MAX-API/dto" "github.com/MAX-API-Next/MAX-API/logger" @@ -36,7 +35,6 @@ func cfStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Res helper.SetEventStreamHeaders(c) id := helper.GetResponseID(c) var responseText string - isFirst := true for scanner.Scan() { data := scanner.Text() @@ -63,12 +61,12 @@ func cfStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Res response.Id = id response.Model = info.UpstreamModelName err = helper.ObjectData(c, response) - if isFirst { - isFirst = false - info.FirstResponseTime = time.Now() - } if err != nil { logger.LogError(c, "error_rendering_stream_response: "+err.Error()) + continue + } + if len(response.Choices) > 0 { + info.SetFirstResponseTime() } } @@ -115,7 +113,9 @@ func cfHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Response) } c.Writer.Header().Set("Content-Type", "application/json") c.Writer.WriteHeader(resp.StatusCode) - _, _ = c.Writer.Write(jsonResponse) + if _, err := c.Writer.Write(jsonResponse); err == nil && len(response.Choices) > 0 { + info.SetFirstResponseTime() + } return nil, usage } @@ -141,7 +141,9 @@ func cfSTTHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Respon } c.Writer.Header().Set("Content-Type", "application/json") c.Writer.WriteHeader(resp.StatusCode) - _, _ = c.Writer.Write(jsonResponse) + if _, err := c.Writer.Write(jsonResponse); err == nil { + info.SetFirstResponseTime() + } usage := service.ResponseText2Usage(c, cfResp.Result.Text, info.UpstreamModelName, info.GetEstimatePromptTokens()) return nil, usage diff --git a/relay/channel/cohere/first_response_test.go b/relay/channel/cohere/first_response_test.go new file mode 100644 index 00000000..528c6911 --- /dev/null +++ b/relay/channel/cohere/first_response_test.go @@ -0,0 +1,82 @@ +package cohere + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + relaycommon "github.com/MAX-API-Next/MAX-API/relay/common" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +func newCohereResponseContext() (*gin.Context, *httptest.ResponseRecorder) { + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(&closeNotifyRecorder{ + ResponseRecorder: recorder, + closeNotify: make(chan bool), + }) + c.Request = httptest.NewRequest(http.MethodGet, "/", nil) + return c, recorder +} + +func newCohereResponseInfo() *relaycommon.RelayInfo { + return &relaycommon.RelayInfo{ + ChannelMeta: &relaycommon.ChannelMeta{ + UpstreamModelName: "cohere-test", + }, + } +} + +func TestCohereStreamHandlerRecordsFirstResponseAfterValidConversion(t *testing.T) { + gin.SetMode(gin.TestMode) + c, recorder := newCohereResponseContext() + info := newCohereResponseInfo() + body := "not-json\n" + `{"text":"hello","is_finished":false}` + "\n" + + usage, apiErr := cohereStreamHandler(c, info, &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(body)), + }) + + require.Nil(t, apiErr) + require.NotNil(t, usage) + require.True(t, info.HasRecordedChannelFirstResponse()) + require.Contains(t, recorder.Body.String(), `"content":"hello"`) +} + +func TestCohereNonStreamingHandlersRecordAfterSuccessfulWrite(t *testing.T) { + gin.SetMode(gin.TestMode) + + t.Run("chat", func(t *testing.T) { + c, _ := newCohereResponseContext() + info := newCohereResponseInfo() + body := `{"response_id":"resp-1","text":"hello","finish_reason":"COMPLETE","meta":{"billed_units":{"input_tokens":1,"output_tokens":2}}}` + + usage, apiErr := cohereHandler(c, info, &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(body)), + }) + + require.Nil(t, apiErr) + require.Equal(t, 3, usage.TotalTokens) + require.True(t, info.HasRecordedChannelFirstResponse()) + }) + + t.Run("rerank", func(t *testing.T) { + c, _ := newCohereResponseContext() + info := newCohereResponseInfo() + body := `{"results":[],"meta":{"billed_units":{"input_tokens":1,"output_tokens":0}}}` + + usage, apiErr := cohereRerankHandler(c, &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(body)), + }, info) + + require.Nil(t, apiErr) + require.Equal(t, 1, usage.TotalTokens) + require.True(t, info.HasRecordedChannelFirstResponse()) + }) +} diff --git a/relay/channel/cohere/relay-cohere.go b/relay/channel/cohere/relay-cohere.go index 63701019..5f71f09a 100644 --- a/relay/channel/cohere/relay-cohere.go +++ b/relay/channel/cohere/relay-cohere.go @@ -5,7 +5,6 @@ import ( "io" "net/http" "strings" - "time" "github.com/MAX-API-Next/MAX-API/common" "github.com/MAX-API-Next/MAX-API/dto" @@ -128,14 +127,9 @@ func cohereStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http } }() helper.SetEventStreamHeaders(c) - isFirst := true c.Stream(func(w io.Writer) bool { select { case data := <-dataChan: - if isFirst { - isFirst = false - info.FirstResponseTime = time.Now() - } data = strings.TrimSuffix(data, "\r") var cohereResp CohereResponse err := json.Unmarshal([]byte(data), &cohereResp) @@ -178,7 +172,11 @@ func cohereStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http common.SysLog("error marshalling stream response: " + err.Error()) return true } - c.Render(-1, common.CustomEvent{Data: "data: " + string(jsonStr)}) + if err := helper.StringData(c, string(jsonStr)); err != nil { + common.SysLog("error writing stream response: " + err.Error()) + return false + } + info.SetFirstResponseTime() return true case <-stopChan: c.Render(-1, common.CustomEvent{Data: "data: [DONE]"}) @@ -236,6 +234,8 @@ func cohereHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Respo c.Writer.WriteHeader(resp.StatusCode) if _, writeErr := c.Writer.Write(jsonResponse); writeErr != nil { common.SysLog("failed to write Cohere response: " + writeErr.Error()) + } else { + info.SetFirstResponseTime() } return &usage, nil } @@ -275,6 +275,8 @@ func cohereRerankHandler(c *gin.Context, resp *http.Response, info *relaycommon. _, err = c.Writer.Write(jsonResponse) if err != nil { common.SysLog("failed to write Cohere rerank response: " + err.Error()) + } else { + info.SetFirstResponseTime() } return &usage, nil } diff --git a/relay/channel/openai/relay-openai.go b/relay/channel/openai/relay-openai.go index 1076b588..80f6b01a 100644 --- a/relay/channel/openai/relay-openai.go +++ b/relay/channel/openai/relay-openai.go @@ -605,7 +605,6 @@ func OpenaiRealtimeHandler(c *gin.Context, info *relaycommon.RelayInfo) (*types. close(targetClosed) return } - info.SetFirstResponseTime() realtimeEvent := &dto.RealtimeEvent{} err = common.Unmarshal(message, realtimeEvent) if err != nil { @@ -692,6 +691,7 @@ func OpenaiRealtimeHandler(c *gin.Context, info *relaycommon.RelayInfo) (*types. errChan <- fmt.Errorf("error writing to client: %v", err) return } + info.SetFirstResponseTime() } } diff --git a/relay/channel/openai/relay_image.go b/relay/channel/openai/relay_image.go index ab12cad9..8e64080b 100644 --- a/relay/channel/openai/relay_image.go +++ b/relay/channel/openai/relay_image.go @@ -243,9 +243,6 @@ func OpenaiImageJSONAsStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, if created == 0 { created = time.Now().Unix() } - if info != nil { - info.SetFirstResponseTime() - } for _, image := range imageResp.Data { payload := map[string]any{ "type": "image_generation.completed", @@ -269,6 +266,9 @@ func OpenaiImageJSONAsStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, } return &usageResp.Usage, nil } + if info != nil { + info.SetFirstResponseTime() + } } if err := writeOpenaiImageStreamDone(c); err != nil { if info != nil && info.StreamStatus != nil { diff --git a/relay/claude_handler.go b/relay/claude_handler.go index ec93f070..dfa1ea31 100644 --- a/relay/claude_handler.go +++ b/relay/claude_handler.go @@ -182,6 +182,9 @@ func ClaudeHelper(c *gin.Context, info *relaycommon.RelayInfo) (maxAPIError *typ service.ResetStatusCode(maxAPIError, statusCodeMappingStr) return maxAPIError } + if !info.IsStream { + info.SetFirstResponseTime() + } service.PostTextConsumeQuota(c, info, usage.(*dto.Usage), nil) return nil diff --git a/relay/common/channel_health_observation_test.go b/relay/common/channel_health_observation_test.go new file mode 100644 index 00000000..f321f623 --- /dev/null +++ b/relay/common/channel_health_observation_test.go @@ -0,0 +1,40 @@ +package common + +import ( + "testing" + "time" + + maxcommon "github.com/MAX-API-Next/MAX-API/common" + "github.com/stretchr/testify/require" +) + +func TestRelayInfoRecordsOneFirstResponsePerChannelAttempt(t *testing.T) { + oldRecorder := recordChannelHealthObservation + t.Cleanup(func() { recordChannelHealthObservation = oldRecorder }) + + var observations []maxcommon.ChannelHealthObservation + recordChannelHealthObservation = func(observation maxcommon.ChannelHealthObservation) error { + observations = append(observations, observation) + return nil + } + + start := time.Now().Add(-2 * time.Second) + info := &RelayInfo{ + StartTime: start, + IsStream: true, + RetryIndex: 2, + isFirstResponse: true, + channelAttemptStartTime: start, + ChannelMeta: &ChannelMeta{ChannelId: 936}, + } + info.SetFirstResponseTime() + info.SetFirstResponseTime() + info.RecordChannelTimeout("stream_idle_timeout") + + require.Len(t, observations, 2) + require.Equal(t, "first_response", observations[0].Event) + require.Equal(t, "stream_idle_timeout", observations[1].Event) + require.Equal(t, 936, observations[0].ChannelID) + require.Equal(t, 2, observations[0].RetryIndex) + require.True(t, observations[0].AttemptLatencyMS >= 0) +} diff --git a/relay/common/relay_info.go b/relay/common/relay_info.go index a3a4dd89..c7f55930 100644 --- a/relay/common/relay_info.go +++ b/relay/common/relay_info.go @@ -1,19 +1,23 @@ package common import ( + "context" "encoding/json" "errors" "fmt" "strconv" "strings" + "sync/atomic" "time" "github.com/MAX-API-Next/MAX-API/common" "github.com/MAX-API-Next/MAX-API/constant" "github.com/MAX-API-Next/MAX-API/dto" "github.com/MAX-API-Next/MAX-API/pkg/billingexpr" + "github.com/MAX-API-Next/MAX-API/pkg/channelhealth" relayconstant "github.com/MAX-API-Next/MAX-API/relay/constant" "github.com/MAX-API-Next/MAX-API/setting/model_setting" + "github.com/MAX-API-Next/MAX-API/setting/operation_setting" "github.com/MAX-API-Next/MAX-API/types" "github.com/gin-gonic/gin" @@ -75,6 +79,7 @@ type ChannelMeta struct { HeadersOverride map[string]interface{} ChannelSetting dto.ChannelSettings ChannelOtherSettings dto.ChannelOtherSettings + ChannelAutoBan bool UpstreamModelName string IsModelMapped bool SupportStreamOptions bool // 是否支持流式选项 @@ -86,16 +91,19 @@ type TokenCountMeta struct { } type RelayInfo struct { - TokenId int - TokenKey string - TokenGroup string - UserId int - UsingGroup string // 使用的分组，当auto跨分组重试时，会变动 - UserGroup string // 用户所在分组 - TokenUnlimited bool - StartTime time.Time - FirstResponseTime time.Time - isFirstResponse bool + TokenId int + TokenKey string + TokenGroup string + UserId int + UsingGroup string // 使用的分组，当auto跨分组重试时，会变动 + UserGroup string // 用户所在分组 + TokenUnlimited bool + StartTime time.Time + FirstResponseTime time.Time + isFirstResponse bool + channelAttemptStartTime time.Time + channelFirstResponseRecorded atomic.Bool + channelFirstResponseSignal chan struct{} //SendLastReasoningResponse bool IsStream bool IsGeminiBatchEmbedding bool @@ -226,6 +234,7 @@ func (info *RelayInfo) InitChannelMeta(c *gin.Context) { ApiKey: common.GetContextKeyString(c, constant.ContextKeyChannelKey), Organization: c.GetString("channel_organization"), ChannelCreateTime: c.GetInt64("channel_create_time"), + ChannelAutoBan: common.GetContextKeyBool(c, constant.ContextKeyChannelAutoBan), ParamOverride: paramOverride, HeadersOverride: headerOverride, UpstreamModelName: common.GetContextKeyString(c, constant.ContextKeyOriginalModel), @@ -255,6 +264,9 @@ func (info *RelayInfo) InitChannelMeta(c *gin.Context) { } info.ChannelMeta = channelMeta + info.channelAttemptStartTime = time.Now() + info.channelFirstResponseRecorded.Store(false) + info.channelFirstResponseSignal = make(chan struct{}) if model_setting.GetGlobalSettings().PassThroughRequestEnabled || channelMeta.ChannelSetting.PassThroughBodyEnabled { info.ReasoningEffort = "" } else { @@ -746,10 +758,165 @@ func (info *RelayInfo) GetEstimatePromptTokens() int { } func (info *RelayInfo) SetFirstResponseTime() { + if info == nil { + return + } if info.isFirstResponse { info.FirstResponseTime = time.Now() info.isFirstResponse = false } + if !info.channelFirstResponseRecorded.CompareAndSwap(false, true) { + return + } + if info.channelFirstResponseSignal != nil { + close(info.channelFirstResponseSignal) + } + info.recordChannelHealthObservation("first_response") + info.recordChannelHealthSuccess() +} + +// FirstResponseSignal is closed once the current channel attempt has produced +// a first valid, deliverable result. It is used by the stream watchdog without +// exposing the internal deduplication state to relay callers. +func (info *RelayInfo) FirstResponseSignal() <-chan struct{} { + if info == nil { + return nil + } + return info.channelFirstResponseSignal +} + +func (info *RelayInfo) HasRecordedChannelFirstResponse() bool { + return info != nil && info.channelFirstResponseRecorded.Load() +} + +var recordChannelHealthObservation = func(observation common.ChannelHealthObservation) error { + common.EnqueueChannelHealthObservation(observation) + return nil +} + +func (info *RelayInfo) recordChannelHealthObservation(event string) { + if info == nil || info.ChannelMeta == nil || info.ChannelId <= 0 { + return + } + start := info.channelAttemptStartTime + if start.IsZero() { + start = info.StartTime + } + observedAt := time.Now() + latency := observedAt.Sub(start).Milliseconds() + if latency < 0 { + latency = 0 + } + _ = recordChannelHealthObservation(common.ChannelHealthObservation{ + ChannelID: info.ChannelId, + Event: event, + ObservedAt: observedAt, + AttemptLatencyMS: latency, + Stream: info.IsStream, + RetryIndex: info.RetryIndex, + }) +} + +func (info *RelayInfo) channelHealthAttemptID() string { + if info == nil { + return "" + } + attemptID := info.RequestId + if attemptID == "" { + attemptStart := info.channelAttemptStartTime + if attemptStart.IsZero() { + attemptStart = info.StartTime + } + attemptID = fmt.Sprintf("attempt:%d:%d", attemptStart.UnixNano(), info.RetryIndex) + } + return fmt.Sprintf("%s:%d:%d", attemptID, info.ChannelId, info.RetryIndex) +} + +// recordChannelHealthSuccess contributes one valid first-response sample only +// when the administrator enabled the advanced timeout gates. It runs off the +// relay response path so a slow Redis instance cannot delay the client. +func (info *RelayInfo) recordChannelHealthSuccess() { + if info == nil || info.ChannelMeta == nil || info.ChannelId <= 0 || !info.ChannelAutoBan { + return + } + setting := operation_setting.GetMonitorSetting() + if setting == nil || !setting.TimeoutAutoDisableEnabled || + (setting.TimeoutAutoDisableMinimumSamples <= 0 && setting.TimeoutAutoDisableRatioPercent <= 0) { + return + } + requestMode := channelhealth.RequestModeNonStreaming + if info.IsStream { + requestMode = channelhealth.RequestModeStreaming + } + evidence := channelhealth.AttemptEvidence{ + ChannelID: info.ChannelId, + AttemptID: info.channelHealthAttemptID(), + RequestMode: requestMode, + TimeoutKind: channelhealth.TimeoutKindFirstResponse, + Success: true, + AutoBan: info.ChannelAutoBan, + ObservedAt: time.Now(), + } + go func() { + decisionContext, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + _, _ = channelhealth.EvaluateTimeoutGuard(decisionContext, evidence, setting) + }() +} + +// RecordChannelTimeout records a confirmed timeout outcome for the current +// channel attempt. It is intentionally side-effect-free for relay behavior. +func (info *RelayInfo) RecordChannelTimeout(event string) { + info.recordChannelHealthObservation(event) + if info == nil { + return + } + timeoutKind := event + if event == "transport_timeout" || event == "request_timeout" { + if info.IsStream && !info.channelFirstResponseRecorded.Load() { + timeoutKind = channelhealth.TimeoutKindStreamingFirstResult + } else if !info.IsStream { + timeoutKind = channelhealth.TimeoutKindNonStreamingResponse + } + } + if timeoutKind != channelhealth.TimeoutKindStreamingFirstResult && + timeoutKind != channelhealth.TimeoutKindNonStreamingResponse { + return + } + setting := operation_setting.GetMonitorSetting() + if setting == nil { + return + } + requestMode := channelhealth.RequestModeNonStreaming + if info.IsStream { + requestMode = channelhealth.RequestModeStreaming + } + decisionContext, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + result, err := channelhealth.EvaluateTimeoutGuard( + decisionContext, + channelhealth.AttemptEvidence{ + ChannelID: info.ChannelId, + AttemptID: info.channelHealthAttemptID(), + RequestMode: requestMode, + TimeoutKind: timeoutKind, + TimeoutEligible: true, + AutoBan: info.ChannelAutoBan, + ObservedAt: time.Now(), + }, + setting, + ) + if err == nil && result.Transition != "" { + common.PublishChannelHealthRuntimeTransition(common.ChannelHealthRuntimeTransition{ + ChannelID: info.ChannelId, + Transition: result.Transition, + Penalty: result.State.Penalty, + PenaltyUntil: result.State.PenaltyUntil, + RuntimeDisabled: result.State.RuntimeDisabled, + DisabledUntil: result.State.DisabledUntil, + ObservedAt: result.State.LastObservedAt, + }) + } } func (info *RelayInfo) HasSendResponse() bool { diff --git a/relay/compatible_handler.go b/relay/compatible_handler.go index e2e34318..9606d001 100644 --- a/relay/compatible_handler.go +++ b/relay/compatible_handler.go @@ -236,6 +236,9 @@ func TextHelper(c *gin.Context, info *relaycommon.RelayInfo) (maxAPIError *types service.ResetStatusCode(maxApiErr, statusCodeMappingStr) return maxApiErr } + if !info.IsStream { + info.SetFirstResponseTime() + } var containAudioTokens = usage.(*dto.Usage).CompletionTokenDetails.AudioTokens > 0 || usage.(*dto.Usage).PromptTokensDetails.AudioTokens > 0 var containsAudioRatios = ratio_setting.ContainsAudioRatio(info.OriginModelName) || ratio_setting.ContainsAudioCompletionRatio(info.OriginModelName) diff --git a/relay/embedding_handler.go b/relay/embedding_handler.go index f5ac08e7..5ac0a97e 100644 --- a/relay/embedding_handler.go +++ b/relay/embedding_handler.go @@ -89,6 +89,9 @@ func EmbeddingHelper(c *gin.Context, info *relaycommon.RelayInfo) (maxAPIError * service.ResetStatusCode(maxAPIError, statusCodeMappingStr) return maxAPIError } + if !info.IsStream { + info.SetFirstResponseTime() + } service.PostTextConsumeQuota(c, info, usage.(*dto.Usage), nil) return nil } diff --git a/relay/gemini_handler.go b/relay/gemini_handler.go index e6fdc519..aba02498 100644 --- a/relay/gemini_handler.go +++ b/relay/gemini_handler.go @@ -201,6 +201,9 @@ func GeminiHelper(c *gin.Context, info *relaycommon.RelayInfo) (maxAPIError *typ service.ResetStatusCode(openaiErr, statusCodeMappingStr) return openaiErr } + if !info.IsStream { + info.SetFirstResponseTime() + } service.PostTextConsumeQuota(c, info, usage.(*dto.Usage), nil) return nil @@ -302,6 +305,9 @@ func GeminiEmbeddingHandler(c *gin.Context, info *relaycommon.RelayInfo) (maxAPI service.ResetStatusCode(openaiErr, statusCodeMappingStr) return openaiErr } + if !info.IsStream { + info.SetFirstResponseTime() + } service.PostTextConsumeQuota(c, info, usage.(*dto.Usage), nil) return nil diff --git a/relay/helper/stream_first_result_timeout_test.go b/relay/helper/stream_first_result_timeout_test.go new file mode 100644 index 00000000..f812b92d --- /dev/null +++ b/relay/helper/stream_first_result_timeout_test.go @@ -0,0 +1,73 @@ +package helper + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + maxcommon "github.com/MAX-API-Next/MAX-API/common" + "github.com/MAX-API-Next/MAX-API/constant" + relaycommon "github.com/MAX-API-Next/MAX-API/relay/common" + "github.com/MAX-API-Next/MAX-API/setting/operation_setting" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +func TestStreamScannerHandlerEnforcesFirstResultTimeout(t *testing.T) { + originalSetting := *operation_setting.GetMonitorSetting() + *operation_setting.GetMonitorSetting() = operation_setting.MonitorSetting{ + StreamingFirstResultTimeoutSeconds: 1, + } + t.Cleanup(func() { *operation_setting.GetMonitorSetting() = originalSetting }) + + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil) + maxcommon.SetContextKey(c, constant.ContextKeyChannelId, 936) + maxcommon.SetContextKey(c, constant.ContextKeyChannelAutoBan, true) + info := &relaycommon.RelayInfo{IsStream: true} + info.InitChannelMeta(c) + + reader, writer := io.Pipe() + t.Cleanup(func() { _ = writer.Close() }) + started := time.Now() + StreamScannerHandler(c, &http.Response{Body: reader}, info, func(_ string, _ *StreamResult) {}) + + require.GreaterOrEqual(t, time.Since(started), time.Second) + require.Equal(t, relaycommon.StreamEndReasonTimeout, info.StreamStatus.EndReason) + require.False(t, info.HasRecordedChannelFirstResponse()) +} + +func TestStreamScannerHandlerDoesNotTreatUsageOnlyFrameAsFirstResult(t *testing.T) { + originalSetting := *operation_setting.GetMonitorSetting() + *operation_setting.GetMonitorSetting() = operation_setting.MonitorSetting{ + StreamingFirstResultTimeoutSeconds: 1, + } + t.Cleanup(func() { *operation_setting.GetMonitorSetting() = originalSetting }) + + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil) + info := &relaycommon.RelayInfo{IsStream: true} + info.InitChannelMeta(c) + reader, writer := io.Pipe() + t.Cleanup(func() { _ = writer.Close() }) + go func() { + _, _ = io.WriteString(writer, "data: {\"usage\":{\"total_tokens\":1}}\n\n") + time.Sleep(100 * time.Millisecond) + _, _ = io.WriteString(writer, "data: {\"choices\":[{\"delta\":{\"content\":\"ok\"}}]}\n\n") + _ = writer.Close() + }() + + StreamScannerHandler(c, &http.Response{Body: reader}, info, func(data string, sr *StreamResult) { + if strings.Contains(data, `"content"`) { + _, _ = c.Writer.Write([]byte("payload")) + } + }) + + require.True(t, info.HasRecordedChannelFirstResponse()) + require.NotEqual(t, relaycommon.StreamEndReasonTimeout, info.StreamStatus.EndReason) +} diff --git a/relay/helper/stream_result.go b/relay/helper/stream_result.go index c26829d7..2ae8967d 100644 --- a/relay/helper/stream_result.go +++ b/relay/helper/stream_result.go @@ -8,8 +8,11 @@ import ( // to record soft errors, signal fatal stops, or mark normal completion. // StreamScannerHandler checks IsStopped() after each callback invocation. type StreamResult struct { - status *relaycommon.StreamStatus - stopped bool + status *relaycommon.StreamStatus + stopped bool + done bool + delivered bool + initialErrorCount int } func newStreamResult(status *relaycommon.StreamStatus) *StreamResult { @@ -38,15 +41,43 @@ func (r *StreamResult) Stop(err error) { // (e.g., Dify "message_end"). The stream stops after this chunk. func (r *StreamResult) Done() { r.status.SetEndReason(relaycommon.StreamEndReasonDone, nil) + r.done = true r.stopped = true } +// MarkDelivered explicitly marks an event whose output was delivered through +// a path that the scanner cannot observe directly. Normal handlers can rely on +// StreamScannerHandler detecting a positive response-writer byte delta. +func (r *StreamResult) MarkDelivered() { + if r != nil { + r.delivered = true + } +} + +func (r *StreamResult) IsDeliverable() bool { + return r != nil && r.delivered +} + // IsStopped returns whether Stop() or Done() was called during this chunk. func (r *StreamResult) IsStopped() bool { return r.stopped } +// IsSuccessful reports whether the handler accepted the current upstream +// event without a fatal stop or a newly recorded soft conversion error. +func (r *StreamResult) IsSuccessful() bool { + if r == nil || r.status == nil { + return false + } + return (r.done || !r.stopped) && r.status.TotalErrorCount() == r.initialErrorCount +} + // reset clears the per-chunk stopped flag so the object can be reused. func (r *StreamResult) reset() { r.stopped = false + r.done = false + r.delivered = false + if r.status != nil { + r.initialErrorCount = r.status.TotalErrorCount() + } } diff --git a/relay/helper/stream_scanner.go b/relay/helper/stream_scanner.go index 9912d51e..8d41d27c 100644 --- a/relay/helper/stream_scanner.go +++ b/relay/helper/stream_scanner.go @@ -88,6 +88,17 @@ func StreamScannerHandler(c *gin.Context, resp *http.Response, info *relaycommon streamingTimeoutSeconds := constant.StreamingTimeout streamingTimeout := time.Duration(streamingTimeoutSeconds) * time.Second ticker, timeoutChan := newStreamingTimeoutTicker(streamingTimeoutSeconds) + monitorSetting := operation_setting.GetMonitorSetting() + firstResultTimeoutSeconds := 0 + if monitorSetting != nil { + firstResultTimeoutSeconds = monitorSetting.StreamingFirstResultTimeoutSeconds + } + var firstResultTimer *time.Timer + var firstResultTimeoutChan <-chan time.Time + if firstResultTimeoutSeconds > 0 && info.FirstResponseSignal() != nil { + firstResultTimer = time.NewTimer(time.Duration(firstResultTimeoutSeconds) * time.Second) + firstResultTimeoutChan = firstResultTimer.C + } var ( stopChan = make(chan bool, 3) // 增加缓冲区避免阻塞 @@ -120,6 +131,7 @@ func StreamScannerHandler(c *gin.Context, resp *http.Response, info *relaycommon logger.LogDebug(c, "relay max idle conns: %d", common.RelayMaxIdleConns) logger.LogDebug(c, "relay max idle conns per host: %d", common.RelayMaxIdleConnsPerHost) logger.LogDebug(c, "streaming timeout seconds: %d", int64(streamingTimeout.Seconds())) + logger.LogDebug(c, "streaming first result timeout seconds: %d", firstResultTimeoutSeconds) logger.LogDebug(c, "ping interval seconds: %d", int64(pingInterval.Seconds())) cleanup := func() { @@ -132,6 +144,14 @@ func StreamScannerHandler(c *gin.Context, resp *http.Response, info *relaycommon if ticker != nil { ticker.Stop() } + if firstResultTimer != nil { + if !firstResultTimer.Stop() { + select { + case <-firstResultTimer.C: + default: + } + } + } if pingTicker != nil { pingTicker.Stop() } @@ -214,7 +234,13 @@ func StreamScannerHandler(c *gin.Context, resp *http.Response, info *relaycommon writeMutex.Lock() defer writeMutex.Unlock() ExtendWriteDeadline(c) + bytesBefore := c.Writer.Size() dataHandler(data, sr) + bytesAfter := c.Writer.Size() + writerDelivered := bytesAfter > bytesBefore && bytesAfter > 0 + if sr.IsSuccessful() && (sr.IsDeliverable() || writerDelivered) { + info.SetFirstResponseTime() + } }() if sr.IsStopped() { return @@ -267,7 +293,6 @@ func StreamScannerHandler(c *gin.Context, resp *http.Response, info *relaycommon continue } if !strings.HasPrefix(data, "[DONE]") { - info.SetFirstResponseTime() info.ReceivedResponseCount++ select { @@ -293,14 +318,34 @@ func StreamScannerHandler(c *gin.Context, resp *http.Response, info *relaycommon info.StreamStatus.SetEndReason(relaycommon.StreamEndReasonEOF, nil) }) - // 主循环等待完成或超时 - select { - case <-timeoutChan: - info.StreamStatus.SetEndReason(relaycommon.StreamEndReasonTimeout, nil) - case <-stopChan: - // EndReason already set by the goroutine that triggered stopChan - case <-c.Request.Context().Done(): - info.StreamStatus.SetEndReason(relaycommon.StreamEndReasonClientGone, c.Request.Context().Err()) + // 主循环等待完成或超时。The first-result channel disables its timer as + // soon as a handler confirms a deliverable payload; subsequent idle timeouts + // retain their existing diagnostic-only semantics. + firstResultSignal := info.FirstResponseSignal() +waitLoop: + for { + select { + case <-firstResultSignal: + firstResultSignal = nil + firstResultTimeoutChan = nil + continue + case <-firstResultTimeoutChan: + firstResultTimeoutChan = nil + if !info.HasRecordedChannelFirstResponse() { + info.StreamStatus.SetEndReason(relaycommon.StreamEndReasonTimeout, nil) + info.RecordChannelTimeout("streaming_first_result_timeout") + break waitLoop + } + continue + case <-timeoutChan: + info.StreamStatus.SetEndReason(relaycommon.StreamEndReasonTimeout, nil) + info.RecordChannelTimeout("stream_idle_timeout") + case <-stopChan: + // EndReason already set by the goroutine that triggered stopChan + case <-c.Request.Context().Done(): + info.StreamStatus.SetEndReason(relaycommon.StreamEndReasonClientGone, c.Request.Context().Err()) + } + break } cleanup() diff --git a/relay/image_handler.go b/relay/image_handler.go index b9ee753c..30aedec2 100644 --- a/relay/image_handler.go +++ b/relay/image_handler.go @@ -117,6 +117,9 @@ func ImageHelper(c *gin.Context, info *relaycommon.RelayInfo) (maxAPIError *type service.ResetStatusCode(maxAPIError, statusCodeMappingStr) return maxAPIError } + if !info.IsStream { + info.SetFirstResponseTime() + } imageN := uint(1) if request.N != nil { diff --git a/relay/relay_task.go b/relay/relay_task.go index ae61fa53..8bd67afd 100644 --- a/relay/relay_task.go +++ b/relay/relay_task.go @@ -411,6 +411,9 @@ func RelayTaskSubmit(c *gin.Context, info *relaycommon.RelayInfo) (*TaskSubmitRe if strings.TrimSpace(upstreamTaskID) == "" { return nil, service.TaskErrorWrapperLocal(errors.New("task upstream response did not contain a task id"), "missing_upstream_task_id", http.StatusBadGateway) } + if !info.IsStream { + info.SetFirstResponseTime() + } // 11. 提交后计费调整：让适配器根据上游实际返回调整 OtherRatios finalQuota := taskEstimateQuota diff --git a/relay/rerank_handler.go b/relay/rerank_handler.go index 5021b302..5326f8ad 100644 --- a/relay/rerank_handler.go +++ b/relay/rerank_handler.go @@ -101,6 +101,9 @@ func RerankHelper(c *gin.Context, info *relaycommon.RelayInfo) (maxAPIError *typ service.ResetStatusCode(maxAPIError, statusCodeMappingStr) return maxAPIError } + if !info.IsStream { + info.SetFirstResponseTime() + } service.PostTextConsumeQuota(c, info, usage.(*dto.Usage), nil) return nil } diff --git a/relay/responses_handler.go b/relay/responses_handler.go index 50797868..a1bb4eb5 100644 --- a/relay/responses_handler.go +++ b/relay/responses_handler.go @@ -145,6 +145,9 @@ func ResponsesHelper(c *gin.Context, info *relaycommon.RelayInfo) (maxAPIError * service.ResetStatusCode(maxAPIError, statusCodeMappingStr) return maxAPIError } + if !info.IsStream { + info.SetFirstResponseTime() + } usageDto := usage.(*dto.Usage) if info.RelayMode == relayconstant.RelayModeResponsesCompact { diff --git a/router/api-router.go b/router/api-router.go index 27309590..e5dd6f83 100644 --- a/router/api-router.go +++ b/router/api-router.go @@ -245,6 +245,9 @@ func SetApiRouter(router *gin.Engine) { smartOpsRoute.Use(middleware.AdminAuth()) { smartOpsRoute.GET("/alerts", controller.GetSmartOpsAlerts) + smartOpsRoute.GET("/channel-health/policy", controller.GetChannelHealthPolicy) + smartOpsRoute.PUT("/channel-health/policy", middleware.RootAuth(), middleware.CriticalRateLimit(), middleware.UserCriticalRateLimit(), middleware.DisableCache(), anonymousRequestBodyLimit, controller.UpdateChannelHealthPolicy) + smartOpsRoute.POST("/channel-health/channels/:id/recover", middleware.RootAuth(), middleware.CriticalRateLimit(), middleware.UserCriticalRateLimit(), middleware.DisableCache(), controller.RecoverChannelHealthRuntime) smartOpsRoute.GET("/billing-settlements", middleware.DisableCache(), controller.GetBillingSettlementReconciliation) smartOpsRoute.PUT("/billing-settlements/blocking-policy", middleware.RootAuth(), middleware.CriticalRateLimit(), middleware.UserCriticalRateLimit(), middleware.DisableCache(), anonymousRequestBodyLimit, controller.UpdateBillingSettlementBlockingPolicy) smartOpsRoute.POST("/billing-settlements/reviews", middleware.CriticalRateLimit(), middleware.UserCriticalRateLimit(), middleware.DisableCache(), anonymousRequestBodyLimit, controller.ReviewBillingSettlements) diff --git a/service/smart_ops_alert.go b/service/smart_ops_alert.go index d112eab8..4f950d4d 100644 --- a/service/smart_ops_alert.go +++ b/service/smart_ops_alert.go @@ -2,6 +2,7 @@ package service import ( "context" + "crypto/sha256" "errors" "fmt" "hash/fnv" @@ -9,13 +10,17 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/MAX-API-Next/MAX-API/common" "github.com/MAX-API-Next/MAX-API/dto" "github.com/MAX-API-Next/MAX-API/model" + "github.com/MAX-API-Next/MAX-API/pkg/channelhealth" "github.com/MAX-API-Next/MAX-API/setting/billing_reconciliation_setting" + "github.com/MAX-API-Next/MAX-API/setting/operation_setting" "github.com/MAX-API-Next/MAX-API/types" + "github.com/go-redis/redis/v8" "gorm.io/gorm" ) @@ -27,15 +32,26 @@ const manualTaskBillingDefaultNote = "Administrator-approved manual task usage s const manualTaskBillingZeroNote = "Administrator-confirmed zero final quota settlement" const ( - smartOpsAlertStatusFiring = "firing" - smartOpsAlertStatusResolved = "resolved" - smartOpsAlertSeverityWarning = "warning" - smartOpsAlertSampleInterval = 5 * time.Second - smartOpsAlertRequiredSamples = 2 - smartOpsAlertQueueSize = 24 - smartOpsAlertWorkerCount = 3 + smartOpsAlertStatusFiring = "firing" + smartOpsAlertStatusResolved = "resolved" + smartOpsAlertSeverityWarning = "warning" + smartOpsAlertSeverityCritical = "critical" + smartOpsAlertSampleInterval = 5 * time.Second + smartOpsAlertRequiredSamples = 2 + smartOpsAlertQueueSize = 24 + smartOpsAlertWorkerCount = 3 + smartOpsAlertDeliveryTTL = 7 * 24 * time.Hour + smartOpsAlertDeliveryTimeout = 100 * time.Millisecond ) +const smartOpsAlertDeliveryKeyPrefix = "maxapi:smart-ops:v1:delivery:" + +const smartOpsAlertRepeatKeyPrefix = "maxapi:smart-ops:v1:repeat:" + +const smartOpsAlertRepeatReleaseScript = `if redis.call("GET", KEYS[1]) == ARGV[1] then return redis.call("DEL", KEYS[1]) else return 0 end` + +var smartOpsAlertRepeatTokenCounter atomic.Uint64 + var smartOpsAlertRetryDelays = []time.Duration{ 5 * time.Second, 15 * time.Second, @@ -46,15 +62,20 @@ var smartOpsAlertRetryDelays = []time.Duration{ // It intentionally excludes secrets, request bodies, channel keys, and // mutation controls. Component-specific values retain their own units. type SmartOpsAlert struct { - Key string `json:"key"` - Status string `json:"status"` - Severity string `json:"severity"` - Component string `json:"component"` - Node string `json:"node,omitempty"` - CurrentValue float64 `json:"current_value"` - Threshold float64 `json:"threshold"` - ObservedAt time.Time `json:"observed_at"` - Message string `json:"message"` + Key string `json:"key"` + Status string `json:"status"` + Severity string `json:"severity"` + Component string `json:"component"` + Node string `json:"node,omitempty"` + CurrentValue float64 `json:"current_value"` + Threshold float64 `json:"threshold"` + ObservedAt time.Time `json:"observed_at"` + Message string `json:"message"` + DeliveryStatus string `json:"delivery_status,omitempty"` + DeliveryAttempts int `json:"delivery_attempts,omitempty"` + DeliveryUpdatedAt time.Time `json:"delivery_updated_at,omitempty"` + DeliveryError string `json:"delivery_error,omitempty"` + repeatLockToken string } type smartOpsAlertEvent struct { @@ -153,6 +174,14 @@ var smartOpsAlertMonitor struct { var smartOpsAlertNotificationSender = enqueueSmartOpsAlertNotification +func init() { + // The relay package cannot import service without creating a cycle. A + // process-local observer keeps the runtime decision in Redis while allowing + // the existing Smart Operations projector and notification queue to react + // immediately after a timeout transition. + common.SetChannelHealthRuntimeTransitionObserver(handleChannelHealthRuntimeTransition) +} + // StartSmartOpsAlertMonitor starts the in-process detector. It is deliberately // separate from the resource sampler so the existing sampler remains cheap and // the detector can be disabled or stopped independently during shutdown. @@ -317,6 +346,117 @@ func projectSmartOpsActiveAlert(alert SmartOpsAlert) { smartOpsAlertMonitor.active[alert.Key] = alert } +func channelHealthPriorityAlertKey(channelID int) string { + return fmt.Sprintf("channel_timeout_priority_demotion:%d", channelID) +} + +func channelHealthDisabledAlertKey(channelID int) string { + return fmt.Sprintf("channel_timeout_auto_disabled:%d", channelID) +} + +func handleChannelHealthRuntimeTransition(transition common.ChannelHealthRuntimeTransition) { + if transition.ChannelID <= 0 || transition.Transition == "" { + return + } + observedAt := transition.ObservedAt + if observedAt.IsZero() { + observedAt = time.Now() + } + + var alert SmartOpsAlert + switch transition.Transition { + case "priority_demotion_firing", "priority_demotion_updated": + if transition.RuntimeDisabled { + return + } + alert = SmartOpsAlert{ + Key: channelHealthPriorityAlertKey(transition.ChannelID), + Status: smartOpsAlertStatusFiring, + Severity: smartOpsAlertSeverityWarning, + Component: "channel", + Node: smartOpsAlertNodeName(), + CurrentValue: float64(transition.Penalty), + Threshold: float64(transition.Penalty), + ObservedAt: observedAt, + Message: formatChannelHealthPriorityAlertMessage(transition), + } + case "timeout_auto_disabled": + // A critical disablement supersedes the warning for the same channel. + // Remove the old projection before publishing the critical event so the + // cockpit cannot show two active states for one runtime transition. + projectSmartOpsActiveAlert(SmartOpsAlert{ + Key: channelHealthPriorityAlertKey(transition.ChannelID), + Status: smartOpsAlertStatusResolved, + }) + alert = SmartOpsAlert{ + Key: channelHealthDisabledAlertKey(transition.ChannelID), + Status: smartOpsAlertStatusFiring, + Severity: smartOpsAlertSeverityCritical, + Component: "channel", + Node: smartOpsAlertNodeName(), + CurrentValue: 1, + Threshold: 1, + ObservedAt: observedAt, + Message: formatChannelHealthDisabledAlertMessage(transition), + } + case "runtime_recovered": + // Recovery must close both possible channel projections. The critical + // disable alert may have been created before a process restart, while a + // warning can still exist when an operator restores a penalty directly. + projectSmartOpsActiveAlert(SmartOpsAlert{ + Key: channelHealthDisabledAlertKey(transition.ChannelID), + Status: smartOpsAlertStatusResolved, + }) + projectSmartOpsActiveAlert(SmartOpsAlert{ + Key: channelHealthPriorityAlertKey(transition.ChannelID), + Status: smartOpsAlertStatusResolved, + }) + if setting := operation_setting.GetMonitorSetting(); setting != nil && setting.ChannelTimeoutNotificationEnabled { + smartOpsAlertNotificationSender(SmartOpsAlert{ + Key: channelHealthDisabledAlertKey(transition.ChannelID), + Status: smartOpsAlertStatusResolved, + Severity: smartOpsAlertSeverityCritical, + Component: "channel", + Node: smartOpsAlertNodeName(), + ObservedAt: observedAt, + Message: fmt.Sprintf("渠道 %d 已恢复运行态，告警已关闭", transition.ChannelID), + }) + } + return + default: + return + } + + // Projection is always updated, even when external notifications are + // disabled. This keeps the cockpit authoritative and makes notification + // transport an operator preference rather than a state dependency. + projectSmartOpsActiveAlert(alert) + setting := operation_setting.GetMonitorSetting() + if setting != nil && setting.ChannelTimeoutNotificationEnabled { + smartOpsAlertNotificationSender(alert) + } +} + +func formatChannelHealthPriorityAlertMessage(transition common.ChannelHealthRuntimeTransition) string { + message := fmt.Sprintf("渠道 %d 已因上游超时降低运行态优先级，扣减 %d", transition.ChannelID, transition.Penalty) + if !transition.PenaltyUntil.IsZero() { + message += "，预计恢复时间 " + transition.PenaltyUntil.Format(time.RFC3339) + } else { + message += "，当前需人工恢复" + } + return message +} + +func formatChannelHealthDisabledAlertMessage(transition common.ChannelHealthRuntimeTransition) string { + message := fmt.Sprintf("渠道 %d 已因重复上游超时被运行态禁用，源状态 repeated_timeout", transition.ChannelID) + if !transition.DisabledUntil.IsZero() { + message += "，计划探测时间 " + transition.DisabledUntil.Format(time.RFC3339) + } else { + message += "，当前需人工恢复" + } + return message +} + func formatSmartOpsAlertMessage(label string, event *smartOpsAlertEvent) string { if event.Status == smartOpsAlertStatusResolved { return fmt.Sprintf("%s 已恢复：当前 %.1f%%，阈值 %.1f%%", label, event.CurrentValue, event.Threshold) @@ -333,19 +473,150 @@ func smartOpsAlertNodeName() string { } // GetSmartOpsAlerts returns the currently firing alerts for the administrator -// cockpit. It is a process-local projection and may be empty immediately after -// restart until the sampler produces two consecutive samples. +// cockpit. Process-local alerts are merged with the bounded Redis channel +// runtime projection so a process restart does not hide an active channel +// penalty or runtime disablement. func GetSmartOpsAlerts() []SmartOpsAlert { + runtimeAlerts, runtimeAvailable := loadChannelHealthSmartOpsAlerts() smartOpsAlertMonitor.Lock() - defer smartOpsAlertMonitor.Unlock() - alerts := make([]SmartOpsAlert, 0, len(smartOpsAlertMonitor.active)) - for _, alert := range smartOpsAlertMonitor.active { + alertsByKey := make(map[string]SmartOpsAlert, len(smartOpsAlertMonitor.active)+len(runtimeAlerts)) + for key, alert := range smartOpsAlertMonitor.active { + alertsByKey[key] = alert + } + if runtimeAvailable { + runtimeKeys := make(map[string]struct{}, len(runtimeAlerts)) + for _, alert := range runtimeAlerts { + runtimeKeys[alert.Key] = struct{}{} + } + // A priority penalty can expire without emitting a process-local + // transition. Once Redis has answered successfully, its projection is + // authoritative and stale channel alerts must not remain active forever. + for key := range alertsByKey { + if strings.HasPrefix(key, "channel_timeout_") { + if _, ok := runtimeKeys[key]; !ok { + delete(alertsByKey, key) + } + } + } + } + for _, alert := range runtimeAlerts { + alertsByKey[alert.Key] = alert + } + smartOpsAlertMonitor.Unlock() + alerts := make([]SmartOpsAlert, 0, len(alertsByKey)) + for _, alert := range alertsByKey { alerts = append(alerts, alert) } + loadSmartOpsAlertDeliveryStates(alerts) sort.Slice(alerts, func(i, j int) bool { return alerts[i].Key < alerts[j].Key }) return alerts } +func loadChannelHealthSmartOpsAlerts() ([]SmartOpsAlert, bool) { + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + states, err := channelhealth.ListRuntimeStates(ctx) + if err != nil { + // Redis is an optional runtime dependency. Existing process-local alerts + // remain available when the shared projection cannot be read. + return nil, false + } + setting := operation_setting.GetMonitorSetting() + now := time.Now() + alerts := make([]SmartOpsAlert, 0, len(states)*2) + for _, state := range states { + observedAt := state.LastObservedAt + if observedAt.IsZero() { + observedAt = now + } + if state.RuntimeDisabled { + alerts = append(alerts, SmartOpsAlert{ + Key: channelHealthDisabledAlertKey(state.ChannelID), + Status: smartOpsAlertStatusFiring, + Severity: smartOpsAlertSeverityCritical, + Component: "channel", + Node: smartOpsAlertNodeName(), + CurrentValue: float64(channelHealthTimeoutCount(state, setting)), + Threshold: float64(channelHealthDisableThreshold(setting)), + ObservedAt: observedAt, + Message: fmt.Sprintf( + "渠道 %d 已因重复上游超时被运行态禁用，源状态 %s", + state.ChannelID, + fallbackChannelHealthSource(state.ActiveSource), + ), + }) + continue + } + if state.Penalty <= 0 || (!state.PenaltyUntil.IsZero() && !now.Before(state.PenaltyUntil)) { + continue + } + transition := common.ChannelHealthRuntimeTransition{ + ChannelID: state.ChannelID, + Penalty: state.Penalty, + PenaltyUntil: state.PenaltyUntil, + ObservedAt: observedAt, + } + alerts = append(alerts, SmartOpsAlert{ + Key: channelHealthPriorityAlertKey(state.ChannelID), + Status: smartOpsAlertStatusFiring, + Severity: smartOpsAlertSeverityWarning, + Component: "channel", + Node: smartOpsAlertNodeName(), + CurrentValue: float64(state.Penalty), + Threshold: float64(channelHealthPenaltyThreshold(setting)), + ObservedAt: observedAt, + Message: formatChannelHealthPriorityAlertMessage(transition), + }) + } + return alerts, true +} + +func channelHealthTimeoutCount(state channelhealth.RuntimeState, setting *operation_setting.MonitorSetting) int64 { + if setting != nil && setting.TimeoutAutoDisableCountScope == operation_setting.TimeoutAutoDisableCountScopeCombined { + return state.TimeoutCount["combined"] + } + mode := state.LastTimeoutMode + if mode == "" { + // Legacy Redis states do not have last_timeout_mode. Preserve their + // previous behavior until the next countable timeout writes it. + switch state.LastEvent { + case channelhealth.TimeoutKindStreamingFirstResult: + mode = channelhealth.RequestModeStreaming + case channelhealth.TimeoutKindNonStreamingResponse: + mode = channelhealth.RequestModeNonStreaming + } + } + switch mode { + case channelhealth.RequestModeStreaming: + return state.TimeoutCount[channelhealth.RequestModeStreaming] + case channelhealth.RequestModeNonStreaming: + return state.TimeoutCount[channelhealth.RequestModeNonStreaming] + default: + return state.TimeoutCount[channelhealth.RequestModeStreaming] + state.TimeoutCount[channelhealth.RequestModeNonStreaming] + } +} + +func channelHealthDisableThreshold(setting *operation_setting.MonitorSetting) int { + if setting == nil { + return 0 + } + return setting.TimeoutAutoDisableCount +} + +func channelHealthPenaltyThreshold(setting *operation_setting.MonitorSetting) int { + if setting == nil { + return 0 + } + return setting.PriorityDeduction +} + +func fallbackChannelHealthSource(source string) string { + if source == "" { + return "repeated_timeout" + } + return source +} + // GetBillingSettlementReconciliation returns bounded, read-only evidence for // open administrator alerts. It never retries or mutates financial state. func GetBillingSettlementReconciliation(limit int) (model.BillingSettlementReconciliationData, error) { @@ -940,6 +1211,39 @@ type smartOpsAlertNotificationPool struct { waitGroup sync.WaitGroup } +type smartOpsAlertDeliveryProjection struct { + alert SmartOpsAlert + status string + attempts int + detail string +} + +var smartOpsAlertDeliveryProjectionQueue struct { + sync.Once + queue chan smartOpsAlertDeliveryProjection +} + +func startSmartOpsAlertDeliveryProjectionWorker() { + smartOpsAlertDeliveryProjectionQueue.Do(func() { + smartOpsAlertDeliveryProjectionQueue.queue = make(chan smartOpsAlertDeliveryProjection, smartOpsAlertQueueSize) + go func() { + for projection := range smartOpsAlertDeliveryProjectionQueue.queue { + recordSmartOpsAlertDeliveryState(projection.alert, projection.status, projection.attempts, projection.detail) + } + }() + }) +} + +func enqueueSmartOpsAlertDeliveryProjection(projection smartOpsAlertDeliveryProjection) bool { + startSmartOpsAlertDeliveryProjectionWorker() + select { + case smartOpsAlertDeliveryProjectionQueue.queue <- projection: + return true + default: + return false + } +} + func newSmartOpsAlertNotificationPool(workerCount, queueSize int, deliver func(SmartOpsAlert) error) *smartOpsAlertNotificationPool { if workerCount < 1 { workerCount = 1 @@ -969,6 +1273,9 @@ func newSmartOpsAlertNotificationPool(workerCount, queueSize int, deliver func(S func (pool *smartOpsAlertNotificationPool) runWorker(queue <-chan SmartOpsAlert) { defer pool.waitGroup.Done() for alert := range queue { + // The relay/observer path only enqueues. Redis projection writes happen + // here, on a bounded worker, so a slow Redis cannot delay a response. + recordSmartOpsAlertDeliveryState(alert, "queued", 0, "") if err := pool.deliver(alert); err != nil { common.SysLog(fmt.Sprintf("failed to deliver smart ops alert notification: %s", err.Error())) } @@ -1023,6 +1330,16 @@ func enqueueSmartOpsAlertNotification(alert SmartOpsAlert) { }) if !smartOpsAlertNotificationQueue.pool.enqueue(alert) { + if !enqueueSmartOpsAlertDeliveryProjection(smartOpsAlertDeliveryProjection{ + alert: alert, + status: "failed", + detail: "notification_queue_full", + }) { + // Both queues are intentionally bounded. Losing this optional + // diagnostic projection is preferable to creating an unbounded + // goroutine or blocking the relay path. + common.SysLog(fmt.Sprintf("smart ops alert delivery projection queue is full for %s", alert.Key)) + } common.SysLog(fmt.Sprintf("smart ops alert notification queue is full, dropping %s for node %s", alert.Key, alert.Node)) } } @@ -1041,21 +1358,153 @@ func smartOpsAlertNotifyType(alert SmartOpsAlert) string { return fmt.Sprintf("%s:%s:%s:%s", dto.NotifyTypeSmartOpsAlert, alert.Node, alert.Key, alert.Status) } +func smartOpsAlertDeliveryKey(alert SmartOpsAlert) string { + digest := sha256.Sum256([]byte(smartOpsAlertNotifyType(alert))) + return fmt.Sprintf("%s%x", smartOpsAlertDeliveryKeyPrefix, digest) +} + +func smartOpsAlertRepeatKey(alert SmartOpsAlert) string { + digest := sha256.Sum256([]byte(alert.Key)) + return fmt.Sprintf("%s%x", smartOpsAlertRepeatKeyPrefix, digest) +} + +// shouldDeliverSmartOpsAlertNotification applies the configured interval to +// repeated priority-demotion updates. Critical disablement and all recovery +// events always pass through. Redis failure is fail-open so an operational +// warning is not silently lost. +func shouldDeliverSmartOpsAlertNotification(alert *SmartOpsAlert) bool { + if alert == nil || alert.Component != "channel" || alert.Status != smartOpsAlertStatusFiring || + !strings.HasPrefix(alert.Key, "channel_timeout_priority_demotion:") { + return true + } + setting := operation_setting.GetMonitorSetting() + if setting == nil || setting.AlertRepeatIntervalSeconds <= 0 || !common.RedisEnabled || common.RDB == nil { + return true + } + ctx, cancel := context.WithTimeout(context.Background(), smartOpsAlertDeliveryTimeout) + defer cancel() + token := fmt.Sprintf("%s:%d:%d", alert.Node, time.Now().UnixNano(), smartOpsAlertRepeatTokenCounter.Add(1)) + allowed, err := common.RDB.SetNX(ctx, smartOpsAlertRepeatKey(*alert), token, time.Duration(setting.AlertRepeatIntervalSeconds)*time.Second).Result() + if err != nil { + return true + } + if allowed { + alert.repeatLockToken = token + } + return allowed +} + +func clearSmartOpsAlertRepeatKey(alert SmartOpsAlert) { + if alert.Component != "channel" || alert.Status != smartOpsAlertStatusFiring || + !strings.HasPrefix(alert.Key, "channel_timeout_priority_demotion:") || + alert.repeatLockToken == "" || + !common.RedisEnabled || common.RDB == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), smartOpsAlertDeliveryTimeout) + defer cancel() + _, _ = common.RDB.Eval(ctx, smartOpsAlertRepeatReleaseScript, []string{smartOpsAlertRepeatKey(alert)}, alert.repeatLockToken).Result() +} + +// recordSmartOpsAlertDeliveryState stores a bounded delivery projection in +// Redis. It is diagnostic and recoverable; failures must never change the +// alert transition or block the notification worker. +func recordSmartOpsAlertDeliveryState(alert SmartOpsAlert, status string, attempts int, detail string) { + if !common.RedisEnabled || common.RDB == nil || status == "" { + return + } + if attempts < 0 { + attempts = 0 + } + if len(detail) > 512 { + detail = detail[:512] + } + ctx, cancel := context.WithTimeout(context.Background(), smartOpsAlertDeliveryTimeout) + defer cancel() + fields := map[string]interface{}{ + "alert_key": alert.Key, + "alert_status": alert.Status, + "alert_node": alert.Node, + "delivery_status": status, + "attempts": strconv.Itoa(attempts), + "error": detail, + "updated_at": time.Now().UTC().Format(time.RFC3339Nano), + } + pipe := common.RDB.TxPipeline() + pipe.HSet(ctx, smartOpsAlertDeliveryKey(alert), fields) + pipe.Expire(ctx, smartOpsAlertDeliveryKey(alert), smartOpsAlertDeliveryTTL) + _, _ = pipe.Exec(ctx) +} + +func loadSmartOpsAlertDeliveryState(alert *SmartOpsAlert) { + if alert == nil { + return + } + alerts := []SmartOpsAlert{*alert} + loadSmartOpsAlertDeliveryStates(alerts) + *alert = alerts[0] +} + +// loadSmartOpsAlertDeliveryStates reads all delivery projections in one +// bounded Redis pipeline. Delivery metadata is diagnostic, so a failed read +// leaves the alert itself intact and simply omits the optional fields. +func loadSmartOpsAlertDeliveryStates(alerts []SmartOpsAlert) { + if len(alerts) == 0 || !common.RedisEnabled || common.RDB == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), smartOpsAlertDeliveryTimeout) + defer cancel() + pipe := common.RDB.Pipeline() + commands := make([]*redis.StringStringMapCmd, len(alerts)) + for index := range alerts { + commands[index] = pipe.HGetAll(ctx, smartOpsAlertDeliveryKey(alerts[index])) + } + if _, err := pipe.Exec(ctx); err != nil { + return + } + for index, command := range commands { + fields, err := command.Result() + if err != nil { + continue + } + alerts[index].DeliveryStatus = fields["delivery_status"] + alerts[index].DeliveryAttempts, _ = strconv.Atoi(fields["attempts"]) + alerts[index].DeliveryError = fields["error"] + if value := fields["updated_at"]; value != "" { + alerts[index].DeliveryUpdatedAt, _ = time.Parse(time.RFC3339Nano, value) + } + } +} + func deliverSmartOpsAlertNotification(alert SmartOpsAlert) error { + if !shouldDeliverSmartOpsAlertNotification(&alert) { + recordSmartOpsAlertDeliveryState(alert, "skipped_repeat", 0, "alert_repeat_interval_seconds") + return nil + } + recordSmartOpsAlertDeliveryState(alert, "sending", 0, "") var recipients []smartOpsAlertRecipient if err := retrySmartOpsAlertOperation(func() error { var err error recipients, err = smartOpsAlertLoadRecipients() return err }); err != nil { + clearSmartOpsAlertRepeatKey(alert) + recordSmartOpsAlertDeliveryState(alert, "delivery_unknown", len(smartOpsAlertRetryDelays)+1, err.Error()) return fmt.Errorf("failed to load administrator recipients: %w", err) } + if len(recipients) == 0 { + clearSmartOpsAlertRepeatKey(alert) + recordSmartOpsAlertDeliveryState(alert, "skipped_unconfigured", 0, "no enabled administrator notification recipient") + return nil + } notification := smartOpsAlertNotification(alert) var firstErr error + deliveryAttempts := 0 for _, recipient := range recipients { allowed, err := checkSmartOpsAlertNotificationLimit(recipient.ID, notification.Type) if err != nil { + deliveryAttempts++ if firstErr == nil { firstErr = err } @@ -1063,6 +1512,7 @@ func deliverSmartOpsAlertNotification(alert SmartOpsAlert) error { continue } if !allowed { + deliveryAttempts++ err = fmt.Errorf("notification limit exceeded for admin user %d", recipient.ID) if firstErr == nil { firstErr = err @@ -1071,6 +1521,7 @@ func deliverSmartOpsAlertNotification(alert SmartOpsAlert) error { continue } if err := retrySmartOpsAlertOperation(func() error { + deliveryAttempts++ return smartOpsAlertSendToRecipient(recipient, notification) }); err != nil { if firstErr == nil { @@ -1079,6 +1530,12 @@ func deliverSmartOpsAlertNotification(alert SmartOpsAlert) error { common.SysLog(fmt.Sprintf("failed to notify admin user %d for smart ops alert after retries: %s", recipient.ID, err.Error())) } } + if firstErr != nil { + clearSmartOpsAlertRepeatKey(alert) + recordSmartOpsAlertDeliveryState(alert, "failed", deliveryAttempts, firstErr.Error()) + } else { + recordSmartOpsAlertDeliveryState(alert, "sent", deliveryAttempts, "") + } return firstErr } @@ -1120,15 +1577,34 @@ func loadSmartOpsAlertRecipients() ([]smartOpsAlertRecipient, error) { recipients := make([]smartOpsAlertRecipient, 0, len(users)) for _, user := range users { + setting := user.GetSetting() + if !smartOpsAlertRecipientConfigured(user.Email, setting) { + continue + } recipients = append(recipients, smartOpsAlertRecipient{ ID: user.Id, Email: user.Email, - Setting: user.GetSetting(), + Setting: setting, }) } return recipients, nil } +func smartOpsAlertRecipientConfigured(email string, setting dto.UserSetting) bool { + switch setting.NotifyType { + case dto.NotifyTypeWebhook: + return strings.TrimSpace(setting.WebhookUrl) != "" + case dto.NotifyTypeBark: + return strings.TrimSpace(setting.BarkUrl) != "" + case dto.NotifyTypeGotify: + return strings.TrimSpace(setting.GotifyUrl) != "" && strings.TrimSpace(setting.GotifyToken) != "" + case "", dto.NotifyTypeEmail: + return strings.TrimSpace(setting.NotificationEmail) != "" || strings.TrimSpace(email) != "" + default: + return false + } +} + // NotifyAdminUsers broadcasts an operational alert to enabled administrators // using each administrator's existing notification method configuration. func NotifyAdminUsers(data dto.Notify) error { diff --git a/service/smart_ops_alert_delivery_test.go b/service/smart_ops_alert_delivery_test.go new file mode 100644 index 00000000..ee7f0c17 --- /dev/null +++ b/service/smart_ops_alert_delivery_test.go @@ -0,0 +1,183 @@ +package service + +import ( + "context" + "fmt" + "testing" + "time" + + maxcommon "github.com/MAX-API-Next/MAX-API/common" + "github.com/MAX-API-Next/MAX-API/dto" + "github.com/MAX-API-Next/MAX-API/setting/operation_setting" + "github.com/alicebob/miniredis/v2" + "github.com/go-redis/redis/v8" + "github.com/stretchr/testify/require" +) + +func withSmartOpsDeliveryRedis(t *testing.T) { + t.Helper() + server := miniredis.RunT(t) + oldRDB, oldEnabled := maxcommon.RDB, maxcommon.RedisEnabled + maxcommon.RDB = redis.NewClient(&redis.Options{Addr: server.Addr()}) + maxcommon.RedisEnabled = true + t.Cleanup(func() { + _ = maxcommon.RDB.Close() + maxcommon.RDB, maxcommon.RedisEnabled = oldRDB, oldEnabled + }) +} + +func TestSmartOpsAlertDeliveryStateIsTraceableInRedis(t *testing.T) { + withSmartOpsDeliveryRedis(t) + alert := SmartOpsAlert{Key: "channel_timeout_auto_disabled:936", Node: "node-a", Status: smartOpsAlertStatusFiring} + + recordSmartOpsAlertDeliveryState(alert, "queued", 0, "") + queued := alert + loadSmartOpsAlertDeliveryState(&queued) + require.Equal(t, "queued", queued.DeliveryStatus) + require.Zero(t, queued.DeliveryAttempts) + + recordSmartOpsAlertDeliveryState(alert, "failed", 3, "temporary delivery failure") + failed := alert + loadSmartOpsAlertDeliveryState(&failed) + require.Equal(t, "failed", failed.DeliveryStatus) + require.Equal(t, 3, failed.DeliveryAttempts) + require.Equal(t, "temporary delivery failure", failed.DeliveryError) + require.False(t, failed.DeliveryUpdatedAt.IsZero()) +} + +func TestDeliverSmartOpsAlertRecordsSentDeliveryState(t *testing.T) { + withSmartOpsDeliveryRedis(t) + originalDelays := smartOpsAlertRetryDelays + originalLoader := smartOpsAlertLoadRecipients + originalLimit := smartOpsAlertCheckNotificationLimit + originalSender := smartOpsAlertSendToRecipient + t.Cleanup(func() { + smartOpsAlertRetryDelays = originalDelays + smartOpsAlertLoadRecipients = originalLoader + smartOpsAlertCheckNotificationLimit = originalLimit + smartOpsAlertSendToRecipient = originalSender + }) + + smartOpsAlertRetryDelays = []time.Duration{0} + smartOpsAlertLoadRecipients = func() ([]smartOpsAlertRecipient, error) { + return []smartOpsAlertRecipient{{ID: 1}}, nil + } + smartOpsAlertCheckNotificationLimit = func(int, string) (bool, error) { return true, nil } + smartOpsAlertSendToRecipient = func(smartOpsAlertRecipient, dto.Notify) error { return nil } + + alert := SmartOpsAlert{Key: "channel_timeout_priority_demotion:936", Node: "node-a", Status: smartOpsAlertStatusFiring} + require.NoError(t, deliverSmartOpsAlertNotification(alert)) + loadSmartOpsAlertDeliveryState(&alert) + require.Equal(t, "sent", alert.DeliveryStatus) + require.Equal(t, 1, alert.DeliveryAttempts) +} + +func TestDeliverSmartOpsAlertHonorsSharedRepeatInterval(t *testing.T) { + withSmartOpsDeliveryRedis(t) + originalSetting := *operation_setting.GetMonitorSetting() + originalDelays := smartOpsAlertRetryDelays + originalLoader := smartOpsAlertLoadRecipients + originalLimit := smartOpsAlertCheckNotificationLimit + originalSender := smartOpsAlertSendToRecipient + t.Cleanup(func() { + *operation_setting.GetMonitorSetting() = originalSetting + smartOpsAlertRetryDelays = originalDelays + smartOpsAlertLoadRecipients = originalLoader + smartOpsAlertCheckNotificationLimit = originalLimit + smartOpsAlertSendToRecipient = originalSender + }) + + *operation_setting.GetMonitorSetting() = operation_setting.MonitorSetting{ + AlertRepeatIntervalSeconds: 60, + } + smartOpsAlertRetryDelays = []time.Duration{0} + smartOpsAlertLoadRecipients = func() ([]smartOpsAlertRecipient, error) { + return []smartOpsAlertRecipient{{ID: 1, Email: "admin@example.com"}}, nil + } + smartOpsAlertCheckNotificationLimit = func(int, string) (bool, error) { return true, nil } + sendCount := 0 + smartOpsAlertSendToRecipient = func(smartOpsAlertRecipient, dto.Notify) error { + sendCount++ + return nil + } + + alert := SmartOpsAlert{ + Key: "channel_timeout_priority_demotion:936", + Node: "node-a", + Status: smartOpsAlertStatusFiring, + Component: "channel", + } + require.NoError(t, deliverSmartOpsAlertNotification(alert)) + require.NoError(t, deliverSmartOpsAlertNotification(alert)) + require.Equal(t, 1, sendCount) + alertState := alert + loadSmartOpsAlertDeliveryState(&alertState) + require.Equal(t, "skipped_repeat", alertState.DeliveryStatus) +} + +func TestDeliverSmartOpsAlertReleasesRepeatIntervalAfterFailure(t *testing.T) { + withSmartOpsDeliveryRedis(t) + originalSetting := *operation_setting.GetMonitorSetting() + originalDelays := smartOpsAlertRetryDelays + originalLoader := smartOpsAlertLoadRecipients + originalLimit := smartOpsAlertCheckNotificationLimit + originalSender := smartOpsAlertSendToRecipient + t.Cleanup(func() { + *operation_setting.GetMonitorSetting() = originalSetting + smartOpsAlertRetryDelays = originalDelays + smartOpsAlertLoadRecipients = originalLoader + smartOpsAlertCheckNotificationLimit = originalLimit + smartOpsAlertSendToRecipient = originalSender + }) + + *operation_setting.GetMonitorSetting() = operation_setting.MonitorSetting{ + AlertRepeatIntervalSeconds: 60, + } + smartOpsAlertRetryDelays = nil + smartOpsAlertLoadRecipients = func() ([]smartOpsAlertRecipient, error) { + return []smartOpsAlertRecipient{{ID: 1, Email: "admin@example.com"}}, nil + } + smartOpsAlertCheckNotificationLimit = func(int, string) (bool, error) { return true, nil } + sendCount := 0 + smartOpsAlertSendToRecipient = func(smartOpsAlertRecipient, dto.Notify) error { + sendCount++ + if sendCount == 1 { + return fmt.Errorf("temporary delivery failure") + } + return nil + } + + alert := SmartOpsAlert{ + Key: "channel_timeout_priority_demotion:936", + Node: "node-a", + Status: smartOpsAlertStatusFiring, + Component: "channel", + } + require.Error(t, deliverSmartOpsAlertNotification(alert)) + require.NoError(t, deliverSmartOpsAlertNotification(alert)) + require.Equal(t, 2, sendCount) +} + +func TestClearSmartOpsAlertRepeatIntervalKeepsNewerLock(t *testing.T) { + withSmartOpsDeliveryRedis(t) + originalSetting := *operation_setting.GetMonitorSetting() + t.Cleanup(func() { *operation_setting.GetMonitorSetting() = originalSetting }) + *operation_setting.GetMonitorSetting() = operation_setting.MonitorSetting{ + AlertRepeatIntervalSeconds: 60, + } + alert := SmartOpsAlert{ + Key: "channel_timeout_priority_demotion:936", + Node: "node-a", + Status: smartOpsAlertStatusFiring, + Component: "channel", + } + require.True(t, shouldDeliverSmartOpsAlertNotification(&alert)) + require.NotEmpty(t, alert.repeatLockToken) + key := smartOpsAlertRepeatKey(alert) + ctx := context.Background() + require.NoError(t, maxcommon.RDB.Set(ctx, key, "newer-lock", time.Minute).Err()) + clearSmartOpsAlertRepeatKey(alert) + value, err := maxcommon.RDB.Get(ctx, key).Result() + require.NoError(t, err) + require.Equal(t, "newer-lock", value) +} diff --git a/service/smart_ops_alert_test.go b/service/smart_ops_alert_test.go index 2a104ec7..4dc6021a 100644 --- a/service/smart_ops_alert_test.go +++ b/service/smart_ops_alert_test.go @@ -8,9 +8,103 @@ import ( "github.com/MAX-API-Next/MAX-API/common" "github.com/MAX-API-Next/MAX-API/dto" "github.com/MAX-API-Next/MAX-API/model" + "github.com/MAX-API-Next/MAX-API/pkg/channelhealth" + "github.com/MAX-API-Next/MAX-API/setting/operation_setting" "github.com/stretchr/testify/require" ) +func TestChannelHealthRuntimeTransitionProjectsAndNotifies(t *testing.T) { + originalSender := smartOpsAlertNotificationSender + originalSetting := *operation_setting.GetMonitorSetting() + smartOpsAlertMonitor.Lock() + originalActive := smartOpsAlertMonitor.active + smartOpsAlertMonitor.active = make(map[string]SmartOpsAlert) + smartOpsAlertMonitor.Unlock() + t.Cleanup(func() { + smartOpsAlertNotificationSender = originalSender + *operation_setting.GetMonitorSetting() = originalSetting + smartOpsAlertMonitor.Lock() + smartOpsAlertMonitor.active = originalActive + smartOpsAlertMonitor.Unlock() + }) + + *operation_setting.GetMonitorSetting() = operation_setting.MonitorSetting{ + ChannelTimeoutNotificationEnabled: true, + } + var events []SmartOpsAlert + smartOpsAlertNotificationSender = func(alert SmartOpsAlert) { + events = append(events, alert) + } + now := time.Now() + handleChannelHealthRuntimeTransition(common.ChannelHealthRuntimeTransition{ + ChannelID: 936, + Transition: "priority_demotion_firing", + Penalty: 10, + PenaltyUntil: now.Add(time.Minute), + ObservedAt: now, + }) + + alerts := GetSmartOpsAlerts() + require.Len(t, events, 1) + require.Equal(t, smartOpsAlertSeverityWarning, events[0].Severity) + require.Equal(t, channelHealthPriorityAlertKey(936), events[0].Key) + require.Contains(t, alerts, SmartOpsAlert{Key: channelHealthPriorityAlertKey(936), Status: smartOpsAlertStatusFiring, Severity: smartOpsAlertSeverityWarning, Component: "channel", Node: events[0].Node, CurrentValue: 10, Threshold: 10, ObservedAt: events[0].ObservedAt, Message: events[0].Message}) + + // Repeated timeouts update the cockpit projection and enter the bounded + // notification queue; the shared repeat gate decides whether delivery is + // due. + handleChannelHealthRuntimeTransition(common.ChannelHealthRuntimeTransition{ + ChannelID: 936, + Transition: "priority_demotion_updated", + Penalty: 10, + PenaltyUntil: now.Add(2 * time.Minute), + ObservedAt: now.Add(time.Second), + }) + require.Len(t, events, 2) + + handleChannelHealthRuntimeTransition(common.ChannelHealthRuntimeTransition{ + ChannelID: 936, + Transition: "timeout_auto_disabled", + RuntimeDisabled: true, + DisabledUntil: now.Add(30 * time.Minute), + ObservedAt: now.Add(2 * time.Second), + }) + require.Len(t, events, 3) + require.Equal(t, smartOpsAlertSeverityCritical, events[2].Severity) + require.Equal(t, channelHealthDisabledAlertKey(936), events[2].Key) + for _, alert := range GetSmartOpsAlerts() { + require.NotEqual(t, channelHealthPriorityAlertKey(936), alert.Key) + } + + handleChannelHealthRuntimeTransition(common.ChannelHealthRuntimeTransition{ + ChannelID: 936, + Transition: "runtime_recovered", + ObservedAt: now.Add(3 * time.Second), + }) + require.Len(t, events, 4) + require.Equal(t, smartOpsAlertStatusResolved, events[3].Status) + for _, alert := range GetSmartOpsAlerts() { + require.NotEqual(t, channelHealthDisabledAlertKey(936), alert.Key) + require.NotEqual(t, channelHealthPriorityAlertKey(936), alert.Key) + } +} + +func TestChannelHealthTimeoutCountUsesTheLatestModeForSameModePolicy(t *testing.T) { + state := channelhealth.RuntimeState{ + TimeoutCount: map[string]int64{ + channelhealth.RequestModeStreaming: 2, + channelhealth.RequestModeNonStreaming: 5, + }, + LastEvent: channelhealth.TimeoutKindStreamingFirstResult, + } + setting := &operation_setting.MonitorSetting{ + TimeoutAutoDisableCountScope: operation_setting.TimeoutAutoDisableCountScopeSameMode, + } + require.Equal(t, int64(2), channelHealthTimeoutCount(state, setting)) + state.LastEvent = channelhealth.TimeoutKindNonStreamingResponse + require.Equal(t, int64(5), channelHealthTimeoutCount(state, setting)) +} + func TestSmartOpsAlertRequiresConsecutiveSamplesAndNotifiesRecovery(t *testing.T) { state := smartOpsAlertState{} firstAt := time.Unix(100, 0) diff --git a/setting/operation_setting/monitor_setting.go b/setting/operation_setting/monitor_setting.go index 006f83c9..7c2bee3c 100644 --- a/setting/operation_setting/monitor_setting.go +++ b/setting/operation_setting/monitor_setting.go @@ -1,16 +1,36 @@ package operation_setting import ( + "fmt" "os" "strconv" + "strings" "github.com/MAX-API-Next/MAX-API/setting/config" ) type MonitorSetting struct { - AutoTestChannelEnabled bool `json:"auto_test_channel_enabled"` - AutoTestChannelMinutes float64 `json:"auto_test_channel_minutes"` - ChannelTestMode string `json:"channel_test_mode"` + AutoTestChannelEnabled bool `json:"auto_test_channel_enabled"` + AutoTestChannelMinutes float64 `json:"auto_test_channel_minutes"` + ChannelTestMode string `json:"channel_test_mode"` + AutoPriorityDemotionEnabled bool `json:"auto_priority_demotion_enabled"` + StreamingFirstResultTimeoutSeconds int `json:"streaming_first_result_timeout_seconds"` + NonStreamingResponseTimeoutSeconds int `json:"non_streaming_response_timeout_seconds"` + PriorityDeduction int `json:"priority_deduction"` + TimeoutAutoDisableEnabled bool `json:"timeout_auto_disable_enabled"` + TimeoutAutoDisableCount int `json:"timeout_auto_disable_count"` + TimeoutAutoDisableWindowSeconds int `json:"timeout_auto_disable_window_seconds"` + TimeoutAutoDisableCountScope string `json:"timeout_auto_disable_count_scope"` + TimeoutAutoDisableMinimumSamples int `json:"timeout_auto_disable_minimum_samples"` + TimeoutAutoDisableRatioPercent float64 `json:"timeout_auto_disable_ratio_percent"` + TimeoutAutoDisableDurationSeconds int `json:"timeout_auto_disable_duration_seconds"` + TimeoutAutoDisableRecoveryMode string `json:"timeout_auto_disable_recovery_mode"` + TimeoutAutoDisableSuccessfulProbeCount int `json:"timeout_auto_disable_successful_probe_count"` + PenaltyCooldownSeconds int `json:"penalty_cooldown_seconds"` + RecoveryMode string `json:"recovery_mode"` + ExtendOnRepeatTimeout bool `json:"extend_on_repeat_timeout"` + ChannelTimeoutNotificationEnabled bool `json:"channel_timeout_notification_enabled"` + AlertRepeatIntervalSeconds int `json:"alert_repeat_interval_seconds"` } const ( @@ -20,11 +40,43 @@ const ( // 默认配置 var monitorSetting = MonitorSetting{ - AutoTestChannelEnabled: false, - AutoTestChannelMinutes: 10, - ChannelTestMode: ChannelTestModeScheduledAll, + AutoTestChannelEnabled: false, + AutoTestChannelMinutes: 10, + ChannelTestMode: ChannelTestModeScheduledAll, + AutoPriorityDemotionEnabled: false, + StreamingFirstResultTimeoutSeconds: 0, + NonStreamingResponseTimeoutSeconds: 0, + PriorityDeduction: 0, + TimeoutAutoDisableEnabled: false, + TimeoutAutoDisableCount: 5, + TimeoutAutoDisableWindowSeconds: 3600, + TimeoutAutoDisableCountScope: TimeoutAutoDisableCountScopeSameMode, + TimeoutAutoDisableMinimumSamples: 0, + // Advanced sample/ratio gates are opt-in. Keeping the ratio at zero by + // default preserves the low-write count-only mode until an administrator + // explicitly configures a sample gate. + TimeoutAutoDisableRatioPercent: 0, + TimeoutAutoDisableDurationSeconds: 1800, + TimeoutAutoDisableRecoveryMode: TimeoutAutoDisableRecoveryProbe, + TimeoutAutoDisableSuccessfulProbeCount: 1, + PenaltyCooldownSeconds: 600, + RecoveryMode: RecoveryModeAutomatic, + ExtendOnRepeatTimeout: true, + // External delivery is opt-in. Smart Operations still keeps the in-process + // and Redis-backed activity projection when this switch is false. + ChannelTimeoutNotificationEnabled: false, + AlertRepeatIntervalSeconds: 900, } +const ( + TimeoutAutoDisableCountScopeSameMode = "same_mode" + TimeoutAutoDisableCountScopeCombined = "combined" + TimeoutAutoDisableRecoveryProbe = "probe_then_automatic" + TimeoutAutoDisableRecoveryManual = "manual" + RecoveryModeAutomatic = "automatic" + RecoveryModeManual = "manual" +) + func init() { // 注册到全局配置管理器 config.GlobalConfig.Register("monitor_setting", &monitorSetting) @@ -48,5 +100,74 @@ func GetMonitorSetting() *MonitorSetting { if monitorSetting.ChannelTestMode != ChannelTestModePassiveRecovery { monitorSetting.ChannelTestMode = ChannelTestModeScheduledAll } + if monitorSetting.TimeoutAutoDisableCountScope != TimeoutAutoDisableCountScopeCombined { + monitorSetting.TimeoutAutoDisableCountScope = TimeoutAutoDisableCountScopeSameMode + } + if monitorSetting.TimeoutAutoDisableRecoveryMode != TimeoutAutoDisableRecoveryManual { + monitorSetting.TimeoutAutoDisableRecoveryMode = TimeoutAutoDisableRecoveryProbe + } + if monitorSetting.RecoveryMode != RecoveryModeManual { + monitorSetting.RecoveryMode = RecoveryModeAutomatic + } return &monitorSetting } + +// ValidateMonitorSettingOption validates one persisted option without +// requiring a database schema change. Cross-field validation belongs to the +// future atomic policy update endpoint. +func ValidateMonitorSettingOption(key, value string) error { + const prefix = "monitor_setting." + if !strings.HasPrefix(key, prefix) { + return nil + } + field := strings.TrimPrefix(key, prefix) + switch field { + case "auto_priority_demotion_enabled", "timeout_auto_disable_enabled", + "extend_on_repeat_timeout", "channel_timeout_notification_enabled": + if _, err := strconv.ParseBool(value); err != nil { + return fmt.Errorf("%s must be boolean", key) + } + case "streaming_first_result_timeout_seconds", "non_streaming_response_timeout_seconds": + return validateMonitorInt(key, value, 0, 86400) + case "priority_deduction": + return validateMonitorInt(key, value, 0, 1<<31-1) + case "timeout_auto_disable_count": + return validateMonitorInt(key, value, 2, 1<<31-1) + case "timeout_auto_disable_window_seconds": + return validateMonitorInt(key, value, 60, 2592000) + case "timeout_auto_disable_minimum_samples": + return validateMonitorInt(key, value, 0, 1<<31-1) + case "timeout_auto_disable_duration_seconds": + return validateMonitorInt(key, value, 0, 604800) + case "timeout_auto_disable_successful_probe_count": + return validateMonitorInt(key, value, 1, 10) + case "penalty_cooldown_seconds", "alert_repeat_interval_seconds": + return validateMonitorInt(key, value, 0, 2592000) + case "timeout_auto_disable_ratio_percent": + ratio, err := strconv.ParseFloat(strings.TrimSpace(value), 64) + if err != nil || ratio < 0 || ratio > 100 { + return fmt.Errorf("%s must be between 0 and 100", key) + } + case "timeout_auto_disable_count_scope": + if value != TimeoutAutoDisableCountScopeSameMode && value != TimeoutAutoDisableCountScopeCombined { + return fmt.Errorf("%s has an invalid value", key) + } + case "timeout_auto_disable_recovery_mode": + if value != TimeoutAutoDisableRecoveryProbe && value != TimeoutAutoDisableRecoveryManual { + return fmt.Errorf("%s has an invalid value", key) + } + case "recovery_mode": + if value != RecoveryModeAutomatic && value != RecoveryModeManual { + return fmt.Errorf("%s has an invalid value", key) + } + } + return nil +} + +func validateMonitorInt(key, value string, min, max int) error { + parsed, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64) + if err != nil || parsed < int64(min) || parsed > int64(max) { + return fmt.Errorf("%s must be an integer between %d and %d", key, min, max) + } + return nil +} diff --git a/setting/operation_setting/monitor_setting_test.go b/setting/operation_setting/monitor_setting_test.go new file mode 100644 index 00000000..ad9bad81 --- /dev/null +++ b/setting/operation_setting/monitor_setting_test.go @@ -0,0 +1,36 @@ +package operation_setting + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMonitorSettingTimeoutDefaults(t *testing.T) { + setting := GetMonitorSetting() + require.False(t, setting.AutoPriorityDemotionEnabled) + require.Equal(t, 5, setting.TimeoutAutoDisableCount) + require.Equal(t, 3600, setting.TimeoutAutoDisableWindowSeconds) + require.Equal(t, TimeoutAutoDisableCountScopeSameMode, setting.TimeoutAutoDisableCountScope) + require.Equal(t, TimeoutAutoDisableRecoveryProbe, setting.TimeoutAutoDisableRecoveryMode) + require.Equal(t, RecoveryModeAutomatic, setting.RecoveryMode) + require.False(t, setting.ChannelTimeoutNotificationEnabled) +} + +func TestValidateMonitorSettingOption(t *testing.T) { + require.NoError(t, ValidateMonitorSettingOption( + "monitor_setting.streaming_first_result_timeout_seconds", "15", + )) + require.NoError(t, ValidateMonitorSettingOption( + "monitor_setting.timeout_auto_disable_ratio_percent", "50", + )) + require.Error(t, ValidateMonitorSettingOption( + "monitor_setting.timeout_auto_disable_count", "1", + )) + require.Error(t, ValidateMonitorSettingOption( + "monitor_setting.timeout_auto_disable_count_scope", "invalid", + )) + require.Error(t, ValidateMonitorSettingOption( + "monitor_setting.priority_deduction", "-1", + )) +} diff --git a/web/default/src/components/layout/config/smart-ops.config.test.ts b/web/default/src/components/layout/config/smart-ops.config.test.ts index cff8e0f4..91036ada 100644 --- a/web/default/src/components/layout/config/smart-ops.config.test.ts +++ b/web/default/src/components/layout/config/smart-ops.config.test.ts @@ -30,7 +30,12 @@ describe('Smart Operations navigation', () => { assert.deepEqual( group.items.map((item) => item.title), - ['Active Alerts', 'Channel performance', 'Model performance'] + [ + 'Active Alerts', + 'Channel health', + 'Channel performance', + 'Model performance', + ] ) }) @@ -41,6 +46,7 @@ describe('Smart Operations navigation', () => { group.items.map((item) => item.title), [ 'Active Alerts', + 'Channel health', 'Channel performance', 'Model performance', 'System Info', diff --git a/web/default/src/components/layout/config/smart-ops.config.ts b/web/default/src/components/layout/config/smart-ops.config.ts index d4213662..636352bf 100644 --- a/web/default/src/components/layout/config/smart-ops.config.ts +++ b/web/default/src/components/layout/config/smart-ops.config.ts @@ -17,7 +17,7 @@ along with this program. If not, see . For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues */ import { type TFunction } from 'i18next' -import { Bot, Gauge, ServerCog, TriangleAlert } from 'lucide-react' +import { Activity, Bot, Gauge, ServerCog, TriangleAlert } from 'lucide-react' import { useAuthStore } from '@/stores/auth-store' import { ROLE } from '@/lib/roles' import type { NavGroup, SidebarView } from '../types' @@ -33,6 +33,11 @@ export function buildSmartOpsNavGroups( url: '/smart-ops/alerts' as const, icon: TriangleAlert, }, + { + title: t('Channel health'), + url: '/smart-ops/channel-health' as const, + icon: Activity, + }, { title: t('Channel performance'), url: '/smart-ops/channel-performance' as const, diff --git a/web/default/src/features/smart-ops/alerts.tsx b/web/default/src/features/smart-ops/alerts.tsx index bc4b8fb2..1a55a34a 100644 --- a/web/default/src/features/smart-ops/alerts.tsx +++ b/web/default/src/features/smart-ops/alerts.tsx @@ -68,6 +68,13 @@ const RESOURCE_LABEL: Record = { system_memory: 'Memory usage', system_disk: 'Disk usage', billing_settlement_backlog: 'Billing reconciliation backlog', + channel_timeout_priority_demotion: 'Channel priority demotion', + channel_timeout_auto_disabled: 'Channel automatically disabled', +} + +function getAlertResourceLabel(key: string): string { + const baseKey = key.split(':', 1)[0] + return RESOURCE_LABEL[baseKey] ?? key } interface ActiveAlertsTableProps { @@ -104,7 +111,7 @@ function ActiveAlertsTable(props: ActiveAlertsTableProps): ReactElement {
- {t(RESOURCE_LABEL[alert.key] ?? alert.key)} + {t(getAlertResourceLabel(alert.key))} {alert.key} @@ -112,7 +119,15 @@ function ActiveAlertsTable(props: ActiveAlertsTableProps): ReactElement {
- {t('Firing')} + + {t('Firing')} + {alert.node || t('Unknown')} @@ -187,7 +202,7 @@ function ActiveAlertsContent(props: ActiveAlertsContentProps): ReactElement { {t('No active alerts.')} {t( - 'This process is not currently reporting sustained host pressure or an open billing reconciliation alert.' + 'This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts.' )} diff --git a/web/default/src/features/smart-ops/api.ts b/web/default/src/features/smart-ops/api.ts index d5b5e272..cd6d65a2 100644 --- a/web/default/src/features/smart-ops/api.ts +++ b/web/default/src/features/smart-ops/api.ts @@ -34,6 +34,10 @@ import type { ModelPerformanceQuery, ModelPerformanceResponse, SmartOpsAlertsResponse, + ChannelHealthPolicyResponse, + ChannelHealthPolicyUpdate, + ChannelHealthPolicyUpdateResponse, + ChannelHealthRecoveryResponse, } from './types' const manualTaskBillingBatchCompletionDataSchema = z.object({ @@ -73,6 +77,35 @@ export async function getSmartOpsAlerts(): Promise { return response.data } +export async function getChannelHealthPolicy(): Promise { + const response = await api.get( + '/api/smart-ops/channel-health/policy' + ) + return response.data +} + +export async function updateChannelHealthPolicy( + updates: ChannelHealthPolicyUpdate[] +): Promise { + const response = await api.put( + '/api/smart-ops/channel-health/policy', + { updates }, + { skipBusinessError: true, skipErrorHandler: true } + ) + return response.data +} + +export async function recoverChannelHealthChannel( + channelId: string +): Promise { + const response = await api.post( + `/api/smart-ops/channel-health/channels/${encodeURIComponent(channelId)}/recover`, + undefined, + { skipBusinessError: true, skipErrorHandler: true } + ) + return response.data +} + export async function getBillingSettlementReconciliation(): Promise { const response = await api.get( '/api/smart-ops/billing-settlements' diff --git a/web/default/src/features/smart-ops/channel-health/index.tsx b/web/default/src/features/smart-ops/channel-health/index.tsx new file mode 100644 index 00000000..0ca04739 --- /dev/null +++ b/web/default/src/features/smart-ops/channel-health/index.tsx @@ -0,0 +1,1009 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import { useMemo, useState, type ReactElement } from 'react' +import { useMutation, useQuery } from '@tanstack/react-query' +import { + Activity, + AlertTriangle, + Bell, + CheckCircle2, + RefreshCw, + RotateCcw, + Save, + ShieldAlert, +} from 'lucide-react' +import { useTranslation } from 'react-i18next' +import { toast } from 'sonner' +import { useAuthStore } from '@/stores/auth-store' +import { ROLE } from '@/lib/roles' +import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert' +import { Badge } from '@/components/ui/badge' +import { Button } from '@/components/ui/button' +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from '@/components/ui/card' +import { Input } from '@/components/ui/input' +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select' +import { Skeleton } from '@/components/ui/skeleton' +import { Switch } from '@/components/ui/switch' +import { ErrorState } from '@/components/error-state' +import { SectionPageLayout } from '@/components/layout' +import { getOptionValue } from '@/features/system-settings/hooks/use-system-options' +import { + getChannelHealthPolicy, + recoverChannelHealthChannel, + getSmartOpsAlerts, + updateChannelHealthPolicy, +} from '../api' +import { getObservedAtMilliseconds } from '../lib/format' +import type { SmartOpsAlert } from '../types' + +const ALERT_POLL_INTERVAL_MS = 30000 + +const DEFAULT_OPTION_VALUES = { + 'monitor_setting.auto_priority_demotion_enabled': false, + 'monitor_setting.streaming_first_result_timeout_seconds': 0, + 'monitor_setting.non_streaming_response_timeout_seconds': 0, + 'monitor_setting.priority_deduction': 0, + 'monitor_setting.timeout_auto_disable_enabled': false, + 'monitor_setting.timeout_auto_disable_count': 5, + 'monitor_setting.timeout_auto_disable_window_seconds': 3600, + 'monitor_setting.timeout_auto_disable_minimum_samples': 0, + 'monitor_setting.timeout_auto_disable_ratio_percent': 0, + 'monitor_setting.timeout_auto_disable_duration_seconds': 1800, + 'monitor_setting.timeout_auto_disable_successful_probe_count': 1, + 'monitor_setting.timeout_auto_disable_recovery_mode': 'probe_then_automatic', + 'monitor_setting.timeout_auto_disable_count_scope': 'same_mode', + 'monitor_setting.penalty_cooldown_seconds': 600, + 'monitor_setting.recovery_mode': 'automatic', + 'monitor_setting.extend_on_repeat_timeout': true, + 'monitor_setting.channel_timeout_notification_enabled': false, + 'monitor_setting.alert_repeat_interval_seconds': 900, +} as const + +type ChannelHealthDraft = { + autoPriorityDemotionEnabled: boolean + streamingFirstResultTimeoutSeconds: number + nonStreamingResponseTimeoutSeconds: number + priorityDeduction: number + timeoutAutoDisableEnabled: boolean + timeoutAutoDisableCount: number + timeoutAutoDisableWindowSeconds: number + timeoutAutoDisableMinimumSamples: number + timeoutAutoDisableRatioPercent: number + timeoutAutoDisableDurationSeconds: number + timeoutAutoDisableSuccessfulProbeCount: number + timeoutAutoDisableRecoveryMode: string + timeoutAutoDisableCountScope: string + penaltyCooldownSeconds: number + recoveryMode: string + extendOnRepeatTimeout: boolean + channelTimeoutNotificationEnabled: boolean + alertRepeatIntervalSeconds: number +} + +const OPTION_KEYS = { + autoPriorityDemotionEnabled: 'monitor_setting.auto_priority_demotion_enabled', + streamingFirstResultTimeoutSeconds: + 'monitor_setting.streaming_first_result_timeout_seconds', + nonStreamingResponseTimeoutSeconds: + 'monitor_setting.non_streaming_response_timeout_seconds', + priorityDeduction: 'monitor_setting.priority_deduction', + timeoutAutoDisableEnabled: 'monitor_setting.timeout_auto_disable_enabled', + timeoutAutoDisableCount: 'monitor_setting.timeout_auto_disable_count', + timeoutAutoDisableWindowSeconds: + 'monitor_setting.timeout_auto_disable_window_seconds', + timeoutAutoDisableMinimumSamples: + 'monitor_setting.timeout_auto_disable_minimum_samples', + timeoutAutoDisableRatioPercent: + 'monitor_setting.timeout_auto_disable_ratio_percent', + timeoutAutoDisableDurationSeconds: + 'monitor_setting.timeout_auto_disable_duration_seconds', + timeoutAutoDisableSuccessfulProbeCount: + 'monitor_setting.timeout_auto_disable_successful_probe_count', + timeoutAutoDisableRecoveryMode: + 'monitor_setting.timeout_auto_disable_recovery_mode', + timeoutAutoDisableCountScope: + 'monitor_setting.timeout_auto_disable_count_scope', + penaltyCooldownSeconds: 'monitor_setting.penalty_cooldown_seconds', + recoveryMode: 'monitor_setting.recovery_mode', + extendOnRepeatTimeout: 'monitor_setting.extend_on_repeat_timeout', + channelTimeoutNotificationEnabled: + 'monitor_setting.channel_timeout_notification_enabled', + alertRepeatIntervalSeconds: 'monitor_setting.alert_repeat_interval_seconds', +} as const + +function parseDraft( + options: Array<{ key: string; value: string }> | undefined +): ChannelHealthDraft { + const values = getOptionValue(options, DEFAULT_OPTION_VALUES) + return { + autoPriorityDemotionEnabled: + values[OPTION_KEYS.autoPriorityDemotionEnabled], + streamingFirstResultTimeoutSeconds: + values[OPTION_KEYS.streamingFirstResultTimeoutSeconds], + nonStreamingResponseTimeoutSeconds: + values[OPTION_KEYS.nonStreamingResponseTimeoutSeconds], + priorityDeduction: values[OPTION_KEYS.priorityDeduction], + timeoutAutoDisableEnabled: values[OPTION_KEYS.timeoutAutoDisableEnabled], + timeoutAutoDisableCount: values[OPTION_KEYS.timeoutAutoDisableCount], + timeoutAutoDisableWindowSeconds: + values[OPTION_KEYS.timeoutAutoDisableWindowSeconds], + timeoutAutoDisableMinimumSamples: + values[OPTION_KEYS.timeoutAutoDisableMinimumSamples], + timeoutAutoDisableRatioPercent: + values[OPTION_KEYS.timeoutAutoDisableRatioPercent], + timeoutAutoDisableDurationSeconds: + values[OPTION_KEYS.timeoutAutoDisableDurationSeconds], + timeoutAutoDisableSuccessfulProbeCount: + values[OPTION_KEYS.timeoutAutoDisableSuccessfulProbeCount], + timeoutAutoDisableRecoveryMode: + values[OPTION_KEYS.timeoutAutoDisableRecoveryMode], + timeoutAutoDisableCountScope: + values[OPTION_KEYS.timeoutAutoDisableCountScope], + penaltyCooldownSeconds: values[OPTION_KEYS.penaltyCooldownSeconds], + recoveryMode: values[OPTION_KEYS.recoveryMode], + extendOnRepeatTimeout: values[OPTION_KEYS.extendOnRepeatTimeout], + channelTimeoutNotificationEnabled: + values[OPTION_KEYS.channelTimeoutNotificationEnabled], + alertRepeatIntervalSeconds: values[OPTION_KEYS.alertRepeatIntervalSeconds], + } +} + +function validateDraft(draft: ChannelHealthDraft): string | null { + const bounded: Array<[number, number, string, number]> = [ + [ + draft.streamingFirstResultTimeoutSeconds, + 0, + 'Streaming first-result timeout must be between 0 and 86400 seconds.', + 86400, + ], + [ + draft.nonStreamingResponseTimeoutSeconds, + 0, + 'Non-streaming response timeout must be between 0 and 86400 seconds.', + 86400, + ], + [ + draft.priorityDeduction, + 0, + 'Priority deduction must be a non-negative integer.', + 2147483647, + ], + [ + draft.timeoutAutoDisableCount, + 2, + 'Disable count must be at least 2.', + 2147483647, + ], + [ + draft.timeoutAutoDisableWindowSeconds, + 60, + 'Disable window must be between 60 and 2592000 seconds.', + 2592000, + ], + [ + draft.timeoutAutoDisableMinimumSamples, + 0, + 'Minimum samples must be a non-negative integer.', + 2147483647, + ], + [ + draft.timeoutAutoDisableDurationSeconds, + 0, + 'Disable duration must be between 0 and 604800 seconds.', + 604800, + ], + [ + draft.timeoutAutoDisableSuccessfulProbeCount, + 1, + 'Successful probe count must be between 1 and 10.', + 10, + ], + [ + draft.penaltyCooldownSeconds, + 0, + 'Penalty cooldown must be between 0 and 2592000 seconds.', + 2592000, + ], + [ + draft.alertRepeatIntervalSeconds, + 0, + 'Alert repeat interval must be between 0 and 2592000 seconds.', + 2592000, + ], + ] + for (const [value, min, message, max] of bounded) { + if (!Number.isInteger(value) || value < min || value > max) { + return message + } + } + if ( + !Number.isFinite(draft.timeoutAutoDisableRatioPercent) || + draft.timeoutAutoDisableRatioPercent < 0 || + draft.timeoutAutoDisableRatioPercent > 100 + ) { + return 'Timeout ratio must be between 0 and 100 percent.' + } + if ( + draft.timeoutAutoDisableRecoveryMode !== 'probe_then_automatic' && + draft.timeoutAutoDisableRecoveryMode !== 'manual' + ) { + return 'Recovery mode is invalid.' + } + if ( + draft.timeoutAutoDisableCountScope !== 'same_mode' && + draft.timeoutAutoDisableCountScope !== 'combined' + ) { + return 'Timeout count scope is invalid.' + } + if (draft.recoveryMode !== 'automatic' && draft.recoveryMode !== 'manual') { + return 'Recovery mode is invalid.' + } + return null +} + +function isChannelAlert(alert: SmartOpsAlert): boolean { + return ( + alert.key.startsWith('channel_timeout_priority_demotion:') || + alert.key.startsWith('channel_timeout_auto_disabled:') + ) +} + +function channelIdFromAlert(alert: SmartOpsAlert): string { + return alert.key.split(':', 2)[1] ?? alert.node ?? '—' +} + +function NumberSetting({ + id, + label, + description, + value, + disabled, + step = 1, + onChange, +}: { + id: string + label: string + description: string + value: number + disabled: boolean + step?: number + onChange: (value: number) => void +}): ReactElement { + return ( + + ) +} + +function SelectSetting({ + label, + description, + value, + disabled, + options, + onChange, +}: { + label: string + description: string + value: string + disabled: boolean + options: Array<{ value: string; label: string }> + onChange: (value: string) => void +}): ReactElement { + return ( + + ) +} + +export function ChannelHealth(): ReactElement { + const { t, i18n } = useTranslation() + const userRole = useAuthStore((state) => state.auth.user?.role) + const canEdit = userRole === ROLE.SUPER_ADMIN + const loadErrorMessage = t('We could not load active alerts.') + const alertsQuery = useQuery({ + queryKey: ['smart-ops', 'channel-health-alerts', loadErrorMessage], + queryFn: async (): Promise => { + const response = await getSmartOpsAlerts() + if (!response.success || !Array.isArray(response.data)) { + throw new Error(response.message || loadErrorMessage) + } + return response.data + }, + retry: false, + refetchInterval: ALERT_POLL_INTERVAL_MS, + }) + const policyQuery = useQuery({ + queryKey: ['smart-ops', 'channel-health-policy'], + queryFn: async () => { + const response = await getChannelHealthPolicy() + const hasAllOptions = + response.success && + Array.isArray(response.data) && + Object.values(OPTION_KEYS).every((key) => + response.data.some((option) => option.key === key) + ) + if (!hasAllOptions) { + throw new Error( + response.message || t('We could not load the channel health policy.') + ) + } + return response + }, + retry: false, + staleTime: 30 * 1000, + }) + const updatePolicy = useMutation({ + mutationFn: updateChannelHealthPolicy, + onSuccess: (response) => { + if (response.success) { + toast.success(t('Setting updated successfully')) + void policyQuery.refetch() + } else { + toast.error(response.message || t('Failed to update setting')) + } + }, + onError: (error: Error) => { + toast.error(error.message || t('Failed to update setting')) + }, + }) + const recoveryMutation = useMutation({ + mutationFn: recoverChannelHealthChannel, + onSuccess: (response) => { + if (response.success) { + toast.success(t('Channel recovery requested')) + void alertsQuery.refetch() + } else { + toast.error(response.message || t('Channel recovery failed')) + } + }, + onError: (error: Error) => { + toast.error(error.message || t('Channel recovery failed')) + }, + }) + const loadedDraft = useMemo( + () => parseDraft(policyQuery.data?.data), + [policyQuery.data?.data] + ) + const [draftOverride, setDraftOverride] = useState( + null + ) + const draft = draftOverride ?? loadedDraft + const updateDraft = ( + updater: (current: ChannelHealthDraft) => ChannelHealthDraft + ) => { + setDraftOverride((current) => updater(current ?? loadedDraft)) + } + + const channelAlerts = (alertsQuery.data ?? []).filter(isChannelAlert) + const demotionAlerts = channelAlerts.filter((alert) => + alert.key.startsWith('channel_timeout_priority_demotion:') + ) + const disabledAlerts = channelAlerts.filter((alert) => + alert.key.startsWith('channel_timeout_auto_disabled:') + ) + const dateFormatter = useMemo( + () => + new Intl.DateTimeFormat(i18n.language, { + dateStyle: 'medium', + timeStyle: 'medium', + }), + [i18n.language] + ) + + const save = async () => { + const validationError = validateDraft(draft) + if (validationError) { + toast.error(t(validationError)) + return + } + const updates: Array<{ key: string; value: string | number | boolean }> = [] + ;(Object.keys(OPTION_KEYS) as Array).forEach( + (name) => { + if (draft[name] !== loadedDraft[name]) { + updates.push({ key: OPTION_KEYS[name], value: draft[name] }) + } + } + ) + if (updates.length === 0) { + return + } + await updatePolicy.mutateAsync(updates) + } + + const setNumber = (name: keyof ChannelHealthDraft, value: number) => + updateDraft((current) => ({ ...current, [name]: value })) + + if (policyQuery.isError) { + return ( + + {t('Channel health')} + + void policyQuery.refetch()} + /> + + + ) + } + + if (policyQuery.isLoading) { + return ( + + {t('Channel health')} + + + + + ) + } + + return ( + + + + {t('Channel health')} + {t('Smart Operations')} + + + + + + +
+ + + +
+ + + + +
+ + + + {t('Timeout policy')} + + {canEdit + ? t( + 'Super administrators can change the existing monitor_setting values.' + ) + : t('Only super administrators can change these values.')} + + + +
+
+
+ {t('Enable automatic priority demotion')} +
+
+ {t( + 'Reduce a channel priority after an eligible upstream timeout.' + )} +
+
+ + updateDraft((current) => ({ + ...current, + autoPriorityDemotionEnabled: checked, + })) + } + /> +
+
+
+
+ {t('Enable automatic channel disable')} +
+
+ {t( + 'Disable a channel when its configured timeout count and ratio gates are met.' + )} +
+
+ + updateDraft((current) => ({ + ...current, + timeoutAutoDisableEnabled: checked, + })) + } + /> +
+
+ + setNumber('streamingFirstResultTimeoutSeconds', value) + } + /> + + setNumber('nonStreamingResponseTimeoutSeconds', value) + } + /> + setNumber('priorityDeduction', value)} + /> + + setNumber('penaltyCooldownSeconds', value) + } + /> + + setNumber('alertRepeatIntervalSeconds', value) + } + /> + + updateDraft((current) => ({ + ...current, + recoveryMode: value, + })) + } + /> + + setNumber('timeoutAutoDisableCount', value) + } + /> + + setNumber('timeoutAutoDisableWindowSeconds', value) + } + /> + + setNumber('timeoutAutoDisableMinimumSamples', value) + } + /> + + setNumber('timeoutAutoDisableRatioPercent', value) + } + /> + + setNumber('timeoutAutoDisableDurationSeconds', value) + } + /> + + setNumber('timeoutAutoDisableSuccessfulProbeCount', value) + } + /> + + updateDraft((current) => ({ + ...current, + timeoutAutoDisableRecoveryMode: value, + })) + } + /> +
+
+
+ {t('Extend penalty on repeated timeout')} +
+
+ {t( + 'Keep the current penalty deadline when repeated timeouts continue during an active penalty.' + )} +
+
+ + updateDraft((current) => ({ + ...current, + extendOnRepeatTimeout: checked, + })) + } + /> +
+ + updateDraft((current) => ({ + ...current, + timeoutAutoDisableCountScope: value, + })) + } + /> +
+
+
+
+ {t('Channel timeout notifications')} +
+
+ {t( + 'Expose channel demotion and disable transitions to the SmartOps alert stream.' + )} +
+
+ + updateDraft((current) => ({ + ...current, + channelTimeoutNotificationEnabled: checked, + })) + } + /> +
+ {canEdit && ( +
+ +
+ )} + + + + {alertsQuery.isLoading && ( + + )} + {alertsQuery.isError && ( + void alertsQuery.refetch()} + /> + )} + {!alertsQuery.isLoading && !alertsQuery.isError && ( + + + {t('Channel alert stream')} + + {t( + 'Live channel timeout transitions from the existing SmartOps alert endpoint.' + )} + + + + {channelAlerts.length === 0 ? ( +
+
+ ) : ( +
+ {channelAlerts.map((alert) => { + const observedAt = getObservedAtMilliseconds(alert) + return ( +
+
+
+ + {alert.severity === 'critical' + ? t('Critical') + : t('Warning')} + + + {channelIdFromAlert(alert) === '—' + ? t('Unknown channel') + : t('Channel {{id}}', { + id: channelIdFromAlert(alert), + })} + +
+
+ {alert.message} +
+ {alert.delivery_status && ( +
+ {t('Delivery status')}: {alert.delivery_status} + {alert.delivery_attempts != null + ? ` (${alert.delivery_attempts})` + : ''} +
+ )} +
+ {alert.key} +
+
+ + {canEdit && ( + + )} +
+ ) + })} +
+ )} + + + )} +
+ + + ) +} + +function StatusCard({ + icon: Icon, + label, + value, +}: { + icon: typeof Activity + label: string + value: number | string +}): ReactElement { + return ( + + + {label} + {value} +
+
+ + + ) +} diff --git a/web/default/src/features/smart-ops/lib/format.ts b/web/default/src/features/smart-ops/lib/format.ts index f42f0c34..421eec7e 100644 --- a/web/default/src/features/smart-ops/lib/format.ts +++ b/web/default/src/features/smart-ops/lib/format.ts @@ -22,6 +22,41 @@ import type { SmartOpsAlert } from '../types' const BILLING_BACKLOG_ALERT_KEY = 'billing_settlement_backlog' +function isChannelPriorityAlert(alert: SmartOpsAlert): boolean { + return alert.key.startsWith('channel_timeout_priority_demotion:') +} + +function isChannelDisabledAlert(alert: SmartOpsAlert): boolean { + return alert.key.startsWith('channel_timeout_auto_disabled:') +} + +function formatChannelValue( + alert: SmartOpsAlert, + value: number, + locale: string, + t: TFunction +): string | null { + if (isChannelPriorityAlert(alert)) { + return formatLocalizedCount( + value, + locale, + t, + '{{count}} priority point', + '{{count}} priority points' + ) + } + if (isChannelDisabledAlert(alert)) { + return formatLocalizedCount( + value, + locale, + t, + '{{count}} timeout', + '{{count}} timeouts' + ) + } + return null +} + export function formatLegacyLatency( milliseconds: number | null | undefined ): string { @@ -125,6 +160,8 @@ export function formatAlertCurrentValue( '{{count}} records' ) } + const channelValue = formatChannelValue(alert, alert.current_value, locale, t) + if (channelValue != null) return channelValue return formatPercent(alert.current_value, locale) } @@ -136,5 +173,7 @@ export function formatAlertThreshold( if (isBillingBacklogAlert(alert)) { return formatDurationSeconds(alert.threshold, locale, t) } + const channelValue = formatChannelValue(alert, alert.threshold, locale, t) + if (channelValue != null) return channelValue return formatPercent(alert.threshold, locale) } diff --git a/web/default/src/features/smart-ops/types.ts b/web/default/src/features/smart-ops/types.ts index cd25dbd0..34055099 100644 --- a/web/default/src/features/smart-ops/types.ts +++ b/web/default/src/features/smart-ops/types.ts @@ -26,13 +26,17 @@ import type { export type SmartOpsAlert = { key: string status: 'firing' - severity: 'warning' + severity: 'warning' | 'critical' component: string node?: string current_value: number threshold: number observed_at: string message: string + delivery_status?: string + delivery_attempts?: number + delivery_updated_at?: string + delivery_error?: string } export type SmartOpsAlertsResponse = { @@ -41,6 +45,30 @@ export type SmartOpsAlertsResponse = { data: SmartOpsAlert[] } +export type ChannelHealthPolicyOption = { + key: string + value: string +} + +export type ChannelHealthPolicyResponse = { + success: boolean + message?: string + data: ChannelHealthPolicyOption[] +} + +export type ChannelHealthPolicyUpdate = { + key: string + value: string | number | boolean +} + +export type ChannelHealthPolicyUpdateResponse = ChannelHealthPolicyResponse + +export type ChannelHealthRecoveryResponse = { + success: boolean + message?: string + data?: Record +} + export type BillingSettlementReconciliationItem = { id: number revision: number diff --git a/web/default/src/i18n/locales/en.json b/web/default/src/i18n/locales/en.json index ac5b65dc..bcbb717e 100644 --- a/web/default/src/i18n/locales/en.json +++ b/web/default/src/i18n/locales/en.json @@ -5493,7 +5493,6 @@ "Zhipu": "Zhipu", "Zhipu V4": "Zhipu V4", "Zoom": "Zoom", - "Smart Operations": "Smart Operations", "Smart Operations Center": "Smart Operations Center", "Production performance": "Production performance", "Production performance is not loaded automatically": "Production performance is not loaded automatically", @@ -5585,6 +5584,59 @@ "Throughput data is partial": "Throughput data is partial", "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete.": "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete.", "Active Alerts": "Active Alerts", + "Channel health": "Channel health", + "Smart Operations": "Smart Operations", + "Channel timeout governance": "Channel timeout governance", + "This view combines timeout alerts with the existing monitor_setting policy. Values are stored through the existing system option endpoint; no new database table or field is used.": "This view combines timeout alerts with the existing monitor_setting policy. Values are stored through the existing system option endpoint; no new database table or field is used.", + "Channel timeout alerts": "Channel timeout alerts", + "Priority demotions": "Priority demotions", + "Automatic disable alerts": "Automatic disable alerts", + "Notifications": "Notifications", + "Timeout policy": "Timeout policy", + "Super administrators can change the existing monitor_setting values.": "Super administrators can change the existing monitor_setting values.", + "Only super administrators can change these values.": "Only super administrators can change these values.", + "Enable automatic priority demotion": "Enable automatic priority demotion", + "Enable automatic channel disable": "Enable automatic channel disable", + "Disable a channel when its configured timeout count and ratio gates are met.": "Disable a channel when its configured timeout count and ratio gates are met.", + "Reduce a channel priority after an eligible upstream timeout.": "Reduce a channel priority after an eligible upstream timeout.", + "Streaming first-result timeout": "Streaming first-result timeout", + "Non-streaming response timeout": "Non-streaming response timeout", + "Seconds; 0 disables this threshold.": "Seconds; 0 disables this threshold.", + "Priority deduction": "Priority deduction", + "Administrator-defined priority points per timeout.": "Administrator-defined priority points per timeout.", + "Penalty cooldown": "Penalty cooldown", + "Seconds before the same channel can be penalized again.": "Seconds before the same channel can be penalized again.", + "Alert repeat interval": "Alert repeat interval", + "Seconds between repeated notifications for one channel.": "Seconds between repeated notifications for one channel.", + "Disable after timeout count": "Disable after timeout count", + "Consecutive/window count used by the existing auto-disable policy.": "Consecutive/window count used by the existing auto-disable policy.", + "Auto-disable window": "Auto-disable window", + "Seconds used to evaluate timeout counts.": "Seconds used to evaluate timeout counts.", + "Minimum timeout samples": "Minimum timeout samples", + "Minimum observations before ratio-based disable is eligible.": "Minimum observations before ratio-based disable is eligible.", + "Timeout ratio threshold": "Timeout ratio threshold", + "Percentage threshold for the existing auto-disable policy.": "Percentage threshold for the existing auto-disable policy.", + "Auto-disable duration": "Auto-disable duration", + "Seconds a channel remains disabled before recovery checks.": "Seconds a channel remains disabled before recovery checks.", + "Successful recovery probes": "Successful recovery probes", + "Successful probes required before automatic recovery.": "Successful probes required before automatic recovery.", + "Channel timeout notifications": "Channel timeout notifications", + "Delivery status": "Delivery status", + "Recovery mode": "Recovery mode", + "Automatic": "Automatic", + "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.", + "Extend penalty on repeated timeout": "Extend penalty on repeated timeout", + "Keep the current penalty deadline when repeated timeouts continue during an active penalty.": "Keep the current penalty deadline when repeated timeouts continue during an active penalty.", + "Expose channel demotion and disable transitions to the SmartOps alert stream.": "Expose channel demotion and disable transitions to the SmartOps alert stream.", + "Save timeout policy": "Save timeout policy", + "Channel alert stream": "Channel alert stream", + "Live channel timeout transitions from the existing SmartOps alert endpoint.": "Live channel timeout transitions from the existing SmartOps alert endpoint.", + "No active channel timeout alerts.": "No active channel timeout alerts.", + "Critical": "Critical", + "Unknown channel": "Unknown channel", + "Channel {{id}}": "Channel {{id}}", + "Channel priority demotion": "Channel priority demotion", + "Channel automatically disabled": "Channel automatically disabled", "Alerts requiring administrator attention": "Alerts requiring administrator attention", "SmartOps reports sustained host pressure here for manual investigation and repair.": "SmartOps reports sustained host pressure here for manual investigation and repair.", "Resource": "Resource", @@ -5597,6 +5649,7 @@ "We could not load active alerts.": "We could not load active alerts.", "No active alerts.": "No active alerts.", "This process is not currently reporting a sustained CPU, memory, or disk alert.": "This process is not currently reporting a sustained CPU, memory, or disk alert.", + "This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts.": "This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts.", "Add a password or sign in again with a linked account to continue.": "Add a password or sign in again with a linked account to continue.", "Add a password or sign in again with a linked account to unlock sensitive operations.": "Add a password or sign in again with a linked account to unlock sensitive operations.", "Additional verification required": "Additional verification required", @@ -5746,6 +5799,35 @@ "Enter exact quotas ({{count}})_other": "Enter exact quotas ({{count}})", "Enter an exact final quota for each selected task. Each value is checked against its own reserved quota.": "Enter an exact final quota for each selected task. Each value is checked against its own reserved quota.", "Apply exact settlements": "Apply exact settlements", - "Allowed range: 0 to {{quota}} ({{quotaPerUnit}} quota = $1).": "Allowed range: 0 to {{quota}} ({{quotaPerUnit}} quota = $1)." + "Allowed range: 0 to {{quota}} ({{quotaPerUnit}} quota = $1).": "Allowed range: 0 to {{quota}} ({{quotaPerUnit}} quota = $1).", + "Automatic recovery mode": "Automatic recovery mode", + "Probe then automatic": "Probe then automatic", + "Manual recovery": "Manual recovery", + "Timeout count scope": "Timeout count scope", + "Same request mode": "Same request mode", + "Combined modes": "Combined modes", + "Count streaming and non-streaming timeouts separately or together.": "Count streaming and non-streaming timeouts separately or together.", + "Probe mode recovers after configured successful probes; manual mode requires an administrator.": "Probe mode recovers after configured successful probes; manual mode requires an administrator.", + "Channel recovery failed": "Channel recovery failed", + "Channel recovery requested": "Channel recovery requested", + "Recover channel": "Recover channel", + "We could not load the channel health policy.": "We could not load the channel health policy.", + "{{count}} priority point": "{{count}} priority point", + "{{count}} priority points": "{{count}} priority points", + "{{count}} timeout": "{{count}} timeout", + "{{count}} timeouts": "{{count}} timeouts", + "Streaming first-result timeout must be between 0 and 86400 seconds.": "Streaming first-result timeout must be between 0 and 86400 seconds.", + "Non-streaming response timeout must be between 0 and 86400 seconds.": "Non-streaming response timeout must be between 0 and 86400 seconds.", + "Priority deduction must be a non-negative integer.": "Priority deduction must be a non-negative integer.", + "Disable count must be at least 2.": "Disable count must be at least 2.", + "Disable window must be between 60 and 2592000 seconds.": "Disable window must be between 60 and 2592000 seconds.", + "Minimum samples must be a non-negative integer.": "Minimum samples must be a non-negative integer.", + "Timeout ratio must be between 0 and 100 percent.": "Timeout ratio must be between 0 and 100 percent.", + "Disable duration must be between 0 and 604800 seconds.": "Disable duration must be between 0 and 604800 seconds.", + "Successful probe count must be between 1 and 10.": "Successful probe count must be between 1 and 10.", + "Penalty cooldown must be between 0 and 2592000 seconds.": "Penalty cooldown must be between 0 and 2592000 seconds.", + "Alert repeat interval must be between 0 and 2592000 seconds.": "Alert repeat interval must be between 0 and 2592000 seconds.", + "Recovery mode is invalid.": "Recovery mode is invalid.", + "Timeout count scope is invalid.": "Timeout count scope is invalid." } } diff --git a/web/default/src/i18n/locales/fr.json b/web/default/src/i18n/locales/fr.json index 2af718f4..b0f57117 100644 --- a/web/default/src/i18n/locales/fr.json +++ b/web/default/src/i18n/locales/fr.json @@ -5746,6 +5746,85 @@ "Enter exact quotas ({{count}})_other": "Saisir les quotas exacts ({{count}})", "Enter an exact final quota for each selected task. Each value is checked against its own reserved quota.": "Saisissez un quota final exact pour chaque tâche sélectionnée. Chaque valeur est vérifiée par rapport à son propre quota réservé.", "Apply exact settlements": "Appliquer les règlements exacts", - "Allowed range: 0 to {{quota}} ({{quotaPerUnit}} quota = $1).": "Plage autorisée : de 0 à {{quota}} ({{quotaPerUnit}} unités de quota = 1 $)." + "Allowed range: 0 to {{quota}} ({{quotaPerUnit}} quota = $1).": "Plage autorisée : de 0 à {{quota}} ({{quotaPerUnit}} unités de quota = 1 $).", + "Automatic recovery mode": "Automatic recovery mode", + "Probe then automatic": "Probe then automatic", + "Manual recovery": "Manual recovery", + "Timeout count scope": "Timeout count scope", + "Same request mode": "Same request mode", + "Combined modes": "Combined modes", + "Count streaming and non-streaming timeouts separately or together.": "Count streaming and non-streaming timeouts separately or together.", + "Probe mode recovers after configured successful probes; manual mode requires an administrator.": "Probe mode recovers after configured successful probes; manual mode requires an administrator.", + "Channel recovery failed": "Channel recovery failed", + "Channel recovery requested": "Channel recovery requested", + "Recover channel": "Recover channel", + "We could not load the channel health policy.": "We could not load the channel health policy.", + "Critical": "Critical", + "Channel timeout notifications": "Channel timeout notifications", + "Delivery status": "Delivery status", + "Recovery mode": "Recovery mode", + "Automatic": "Automatic", + "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.", + "Extend penalty on repeated timeout": "Extend penalty on repeated timeout", + "Keep the current penalty deadline when repeated timeouts continue during an active penalty.": "Keep the current penalty deadline when repeated timeouts continue during an active penalty.", + "{{count}} priority point": "{{count}} priority point", + "{{count}} priority points": "{{count}} priority points", + "{{count}} timeout": "{{count}} timeout", + "{{count}} timeouts": "{{count}} timeouts", + "Streaming first-result timeout must be between 0 and 86400 seconds.": "Streaming first-result timeout must be between 0 and 86400 seconds.", + "Non-streaming response timeout must be between 0 and 86400 seconds.": "Non-streaming response timeout must be between 0 and 86400 seconds.", + "Priority deduction must be a non-negative integer.": "Priority deduction must be a non-negative integer.", + "Disable count must be at least 2.": "Disable count must be at least 2.", + "Disable window must be between 60 and 2592000 seconds.": "Disable window must be between 60 and 2592000 seconds.", + "Minimum samples must be a non-negative integer.": "Minimum samples must be a non-negative integer.", + "Timeout ratio must be between 0 and 100 percent.": "Timeout ratio must be between 0 and 100 percent.", + "Disable duration must be between 0 and 604800 seconds.": "Disable duration must be between 0 and 604800 seconds.", + "Successful probe count must be between 1 and 10.": "Successful probe count must be between 1 and 10.", + "Penalty cooldown must be between 0 and 2592000 seconds.": "Penalty cooldown must be between 0 and 2592000 seconds.", + "Alert repeat interval must be between 0 and 2592000 seconds.": "Alert repeat interval must be between 0 and 2592000 seconds.", + "Recovery mode is invalid.": "Recovery mode is invalid.", + "Timeout count scope is invalid.": "Timeout count scope is invalid.", + "Channel health": "Channel health", + "Channel timeout governance": "Channel timeout governance", + "This view combines timeout alerts with the existing monitor_setting policy. Values are stored through the existing system option endpoint; no new database table or field is used.": "This view combines timeout alerts with the existing monitor_setting policy. Values are stored through the existing system option endpoint; no new database table or field is used.", + "Channel timeout alerts": "Channel timeout alerts", + "Priority demotions": "Priority demotions", + "Automatic disable alerts": "Automatic disable alerts", + "Timeout policy": "Timeout policy", + "Super administrators can change the existing monitor_setting values.": "Super administrators can change the existing monitor_setting values.", + "Only super administrators can change these values.": "Only super administrators can change these values.", + "Enable automatic priority demotion": "Enable automatic priority demotion", + "Reduce a channel priority after an eligible upstream timeout.": "Reduce a channel priority after an eligible upstream timeout.", + "Enable automatic channel disable": "Enable automatic channel disable", + "Disable a channel when its configured timeout count and ratio gates are met.": "Disable a channel when its configured timeout count and ratio gates are met.", + "Streaming first-result timeout": "Streaming first-result timeout", + "Seconds; 0 disables this threshold.": "Seconds; 0 disables this threshold.", + "Non-streaming response timeout": "Non-streaming response timeout", + "Priority deduction": "Priority deduction", + "Administrator-defined priority points per timeout.": "Administrator-defined priority points per timeout.", + "Penalty cooldown": "Penalty cooldown", + "Seconds before the same channel can be penalized again.": "Seconds before the same channel can be penalized again.", + "Alert repeat interval": "Alert repeat interval", + "Seconds between repeated notifications for one channel.": "Seconds between repeated notifications for one channel.", + "Disable after timeout count": "Disable after timeout count", + "Consecutive/window count used by the existing auto-disable policy.": "Consecutive/window count used by the existing auto-disable policy.", + "Auto-disable window": "Auto-disable window", + "Seconds used to evaluate timeout counts.": "Seconds used to evaluate timeout counts.", + "Minimum timeout samples": "Minimum timeout samples", + "Minimum observations before ratio-based disable is eligible.": "Minimum observations before ratio-based disable is eligible.", + "Timeout ratio threshold": "Timeout ratio threshold", + "Percentage threshold for the existing auto-disable policy.": "Percentage threshold for the existing auto-disable policy.", + "Auto-disable duration": "Auto-disable duration", + "Seconds a channel remains disabled before recovery checks.": "Seconds a channel remains disabled before recovery checks.", + "Successful recovery probes": "Successful recovery probes", + "Successful probes required before automatic recovery.": "Successful probes required before automatic recovery.", + "Expose channel demotion and disable transitions to the SmartOps alert stream.": "Expose channel demotion and disable transitions to the SmartOps alert stream.", + "Save timeout policy": "Save timeout policy", + "Channel alert stream": "Channel alert stream", + "Live channel timeout transitions from the existing SmartOps alert endpoint.": "Live channel timeout transitions from the existing SmartOps alert endpoint.", + "No active channel timeout alerts.": "No active channel timeout alerts.", + "Unknown channel": "Unknown channel", + "Channel {{id}}": "Channel {{id}}", + "This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts.": "This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts." } } diff --git a/web/default/src/i18n/locales/ja.json b/web/default/src/i18n/locales/ja.json index 2e23fe26..3441232b 100644 --- a/web/default/src/i18n/locales/ja.json +++ b/web/default/src/i18n/locales/ja.json @@ -5746,6 +5746,85 @@ "Enter exact quotas ({{count}})_other": "正確なクォータを入力（{{count}} 件）", "Enter an exact final quota for each selected task. Each value is checked against its own reserved quota.": "選択した各タスクの正確な最終クォータを入力してください。各値はタスク固有の予約クォータと照合されます。", "Apply exact settlements": "正確な精算を適用", - "Allowed range: 0 to {{quota}} ({{quotaPerUnit}} quota = $1).": "許可範囲：0 ～ {{quota}}（{{quotaPerUnit}} クォータ = $1）。" + "Allowed range: 0 to {{quota}} ({{quotaPerUnit}} quota = $1).": "許可範囲：0 ～ {{quota}}（{{quotaPerUnit}} クォータ = $1）。", + "Automatic recovery mode": "Automatic recovery mode", + "Probe then automatic": "Probe then automatic", + "Manual recovery": "Manual recovery", + "Timeout count scope": "Timeout count scope", + "Same request mode": "Same request mode", + "Combined modes": "Combined modes", + "Count streaming and non-streaming timeouts separately or together.": "Count streaming and non-streaming timeouts separately or together.", + "Probe mode recovers after configured successful probes; manual mode requires an administrator.": "Probe mode recovers after configured successful probes; manual mode requires an administrator.", + "Channel recovery failed": "Channel recovery failed", + "Channel recovery requested": "Channel recovery requested", + "Recover channel": "Recover channel", + "We could not load the channel health policy.": "We could not load the channel health policy.", + "Critical": "Critical", + "Channel timeout notifications": "Channel timeout notifications", + "Delivery status": "Delivery status", + "Recovery mode": "Recovery mode", + "Automatic": "Automatic", + "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.", + "Extend penalty on repeated timeout": "Extend penalty on repeated timeout", + "Keep the current penalty deadline when repeated timeouts continue during an active penalty.": "Keep the current penalty deadline when repeated timeouts continue during an active penalty.", + "{{count}} priority point": "{{count}} priority point", + "{{count}} priority points": "{{count}} priority points", + "{{count}} timeout": "{{count}} timeout", + "{{count}} timeouts": "{{count}} timeouts", + "Streaming first-result timeout must be between 0 and 86400 seconds.": "Streaming first-result timeout must be between 0 and 86400 seconds.", + "Non-streaming response timeout must be between 0 and 86400 seconds.": "Non-streaming response timeout must be between 0 and 86400 seconds.", + "Priority deduction must be a non-negative integer.": "Priority deduction must be a non-negative integer.", + "Disable count must be at least 2.": "Disable count must be at least 2.", + "Disable window must be between 60 and 2592000 seconds.": "Disable window must be between 60 and 2592000 seconds.", + "Minimum samples must be a non-negative integer.": "Minimum samples must be a non-negative integer.", + "Timeout ratio must be between 0 and 100 percent.": "Timeout ratio must be between 0 and 100 percent.", + "Disable duration must be between 0 and 604800 seconds.": "Disable duration must be between 0 and 604800 seconds.", + "Successful probe count must be between 1 and 10.": "Successful probe count must be between 1 and 10.", + "Penalty cooldown must be between 0 and 2592000 seconds.": "Penalty cooldown must be between 0 and 2592000 seconds.", + "Alert repeat interval must be between 0 and 2592000 seconds.": "Alert repeat interval must be between 0 and 2592000 seconds.", + "Recovery mode is invalid.": "Recovery mode is invalid.", + "Timeout count scope is invalid.": "Timeout count scope is invalid.", + "Channel health": "Channel health", + "Channel timeout governance": "Channel timeout governance", + "This view combines timeout alerts with the existing monitor_setting policy. Values are stored through the existing system option endpoint; no new database table or field is used.": "This view combines timeout alerts with the existing monitor_setting policy. Values are stored through the existing system option endpoint; no new database table or field is used.", + "Channel timeout alerts": "Channel timeout alerts", + "Priority demotions": "Priority demotions", + "Automatic disable alerts": "Automatic disable alerts", + "Timeout policy": "Timeout policy", + "Super administrators can change the existing monitor_setting values.": "Super administrators can change the existing monitor_setting values.", + "Only super administrators can change these values.": "Only super administrators can change these values.", + "Enable automatic priority demotion": "Enable automatic priority demotion", + "Reduce a channel priority after an eligible upstream timeout.": "Reduce a channel priority after an eligible upstream timeout.", + "Enable automatic channel disable": "Enable automatic channel disable", + "Disable a channel when its configured timeout count and ratio gates are met.": "Disable a channel when its configured timeout count and ratio gates are met.", + "Streaming first-result timeout": "Streaming first-result timeout", + "Seconds; 0 disables this threshold.": "Seconds; 0 disables this threshold.", + "Non-streaming response timeout": "Non-streaming response timeout", + "Priority deduction": "Priority deduction", + "Administrator-defined priority points per timeout.": "Administrator-defined priority points per timeout.", + "Penalty cooldown": "Penalty cooldown", + "Seconds before the same channel can be penalized again.": "Seconds before the same channel can be penalized again.", + "Alert repeat interval": "Alert repeat interval", + "Seconds between repeated notifications for one channel.": "Seconds between repeated notifications for one channel.", + "Disable after timeout count": "Disable after timeout count", + "Consecutive/window count used by the existing auto-disable policy.": "Consecutive/window count used by the existing auto-disable policy.", + "Auto-disable window": "Auto-disable window", + "Seconds used to evaluate timeout counts.": "Seconds used to evaluate timeout counts.", + "Minimum timeout samples": "Minimum timeout samples", + "Minimum observations before ratio-based disable is eligible.": "Minimum observations before ratio-based disable is eligible.", + "Timeout ratio threshold": "Timeout ratio threshold", + "Percentage threshold for the existing auto-disable policy.": "Percentage threshold for the existing auto-disable policy.", + "Auto-disable duration": "Auto-disable duration", + "Seconds a channel remains disabled before recovery checks.": "Seconds a channel remains disabled before recovery checks.", + "Successful recovery probes": "Successful recovery probes", + "Successful probes required before automatic recovery.": "Successful probes required before automatic recovery.", + "Expose channel demotion and disable transitions to the SmartOps alert stream.": "Expose channel demotion and disable transitions to the SmartOps alert stream.", + "Save timeout policy": "Save timeout policy", + "Channel alert stream": "Channel alert stream", + "Live channel timeout transitions from the existing SmartOps alert endpoint.": "Live channel timeout transitions from the existing SmartOps alert endpoint.", + "No active channel timeout alerts.": "No active channel timeout alerts.", + "Unknown channel": "Unknown channel", + "Channel {{id}}": "Channel {{id}}", + "This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts.": "This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts." } } diff --git a/web/default/src/i18n/locales/ru.json b/web/default/src/i18n/locales/ru.json index 2c47d289..ce3a216b 100644 --- a/web/default/src/i18n/locales/ru.json +++ b/web/default/src/i18n/locales/ru.json @@ -5754,6 +5754,85 @@ "This will settle {{displayCount}} selected MiniMax-H3 task(s) with an explicit final quota of 0 and refund the unused reservation. Continue only after verifying the provider result._few": "Будут выполнены расчёты для {{displayCount}} выбранных задач MiniMax-H3 с явной итоговой квотой 0 и возвращён неиспользованный резерв. Продолжайте только после проверки результата провайдера.", "This will settle {{displayCount}} selected MiniMax-H3 task(s) with an explicit final quota of 0 and refund the unused reservation. Continue only after verifying the provider result._many": "Будут выполнены расчёты для {{displayCount}} выбранных задач MiniMax-H3 с явной итоговой квотой 0 и возвращён неиспользованный резерв. Продолжайте только после проверки результата провайдера.", "Enter exact quotas ({{count}})_few": "Ввести точные квоты ({{count}})", - "Enter exact quotas ({{count}})_many": "Ввести точные квоты ({{count}})" + "Enter exact quotas ({{count}})_many": "Ввести точные квоты ({{count}})", + "Automatic recovery mode": "Automatic recovery mode", + "Probe then automatic": "Probe then automatic", + "Manual recovery": "Manual recovery", + "Timeout count scope": "Timeout count scope", + "Same request mode": "Same request mode", + "Combined modes": "Combined modes", + "Count streaming and non-streaming timeouts separately or together.": "Count streaming and non-streaming timeouts separately or together.", + "Probe mode recovers after configured successful probes; manual mode requires an administrator.": "Probe mode recovers after configured successful probes; manual mode requires an administrator.", + "Channel recovery failed": "Channel recovery failed", + "Channel recovery requested": "Channel recovery requested", + "Recover channel": "Recover channel", + "We could not load the channel health policy.": "We could not load the channel health policy.", + "Critical": "Critical", + "Channel timeout notifications": "Channel timeout notifications", + "Delivery status": "Delivery status", + "Recovery mode": "Recovery mode", + "Automatic": "Automatic", + "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.", + "Extend penalty on repeated timeout": "Extend penalty on repeated timeout", + "Keep the current penalty deadline when repeated timeouts continue during an active penalty.": "Keep the current penalty deadline when repeated timeouts continue during an active penalty.", + "{{count}} priority point": "{{count}} priority point", + "{{count}} priority points": "{{count}} priority points", + "{{count}} timeout": "{{count}} timeout", + "{{count}} timeouts": "{{count}} timeouts", + "Streaming first-result timeout must be between 0 and 86400 seconds.": "Streaming first-result timeout must be between 0 and 86400 seconds.", + "Non-streaming response timeout must be between 0 and 86400 seconds.": "Non-streaming response timeout must be between 0 and 86400 seconds.", + "Priority deduction must be a non-negative integer.": "Priority deduction must be a non-negative integer.", + "Disable count must be at least 2.": "Disable count must be at least 2.", + "Disable window must be between 60 and 2592000 seconds.": "Disable window must be between 60 and 2592000 seconds.", + "Minimum samples must be a non-negative integer.": "Minimum samples must be a non-negative integer.", + "Timeout ratio must be between 0 and 100 percent.": "Timeout ratio must be between 0 and 100 percent.", + "Disable duration must be between 0 and 604800 seconds.": "Disable duration must be between 0 and 604800 seconds.", + "Successful probe count must be between 1 and 10.": "Successful probe count must be between 1 and 10.", + "Penalty cooldown must be between 0 and 2592000 seconds.": "Penalty cooldown must be between 0 and 2592000 seconds.", + "Alert repeat interval must be between 0 and 2592000 seconds.": "Alert repeat interval must be between 0 and 2592000 seconds.", + "Recovery mode is invalid.": "Recovery mode is invalid.", + "Timeout count scope is invalid.": "Timeout count scope is invalid.", + "Channel health": "Channel health", + "Channel timeout governance": "Channel timeout governance", + "This view combines timeout alerts with the existing monitor_setting policy. Values are stored through the existing system option endpoint; no new database table or field is used.": "This view combines timeout alerts with the existing monitor_setting policy. Values are stored through the existing system option endpoint; no new database table or field is used.", + "Channel timeout alerts": "Channel timeout alerts", + "Priority demotions": "Priority demotions", + "Automatic disable alerts": "Automatic disable alerts", + "Timeout policy": "Timeout policy", + "Super administrators can change the existing monitor_setting values.": "Super administrators can change the existing monitor_setting values.", + "Only super administrators can change these values.": "Only super administrators can change these values.", + "Enable automatic priority demotion": "Enable automatic priority demotion", + "Reduce a channel priority after an eligible upstream timeout.": "Reduce a channel priority after an eligible upstream timeout.", + "Enable automatic channel disable": "Enable automatic channel disable", + "Disable a channel when its configured timeout count and ratio gates are met.": "Disable a channel when its configured timeout count and ratio gates are met.", + "Streaming first-result timeout": "Streaming first-result timeout", + "Seconds; 0 disables this threshold.": "Seconds; 0 disables this threshold.", + "Non-streaming response timeout": "Non-streaming response timeout", + "Priority deduction": "Priority deduction", + "Administrator-defined priority points per timeout.": "Administrator-defined priority points per timeout.", + "Penalty cooldown": "Penalty cooldown", + "Seconds before the same channel can be penalized again.": "Seconds before the same channel can be penalized again.", + "Alert repeat interval": "Alert repeat interval", + "Seconds between repeated notifications for one channel.": "Seconds between repeated notifications for one channel.", + "Disable after timeout count": "Disable after timeout count", + "Consecutive/window count used by the existing auto-disable policy.": "Consecutive/window count used by the existing auto-disable policy.", + "Auto-disable window": "Auto-disable window", + "Seconds used to evaluate timeout counts.": "Seconds used to evaluate timeout counts.", + "Minimum timeout samples": "Minimum timeout samples", + "Minimum observations before ratio-based disable is eligible.": "Minimum observations before ratio-based disable is eligible.", + "Timeout ratio threshold": "Timeout ratio threshold", + "Percentage threshold for the existing auto-disable policy.": "Percentage threshold for the existing auto-disable policy.", + "Auto-disable duration": "Auto-disable duration", + "Seconds a channel remains disabled before recovery checks.": "Seconds a channel remains disabled before recovery checks.", + "Successful recovery probes": "Successful recovery probes", + "Successful probes required before automatic recovery.": "Successful probes required before automatic recovery.", + "Expose channel demotion and disable transitions to the SmartOps alert stream.": "Expose channel demotion and disable transitions to the SmartOps alert stream.", + "Save timeout policy": "Save timeout policy", + "Channel alert stream": "Channel alert stream", + "Live channel timeout transitions from the existing SmartOps alert endpoint.": "Live channel timeout transitions from the existing SmartOps alert endpoint.", + "No active channel timeout alerts.": "No active channel timeout alerts.", + "Unknown channel": "Unknown channel", + "Channel {{id}}": "Channel {{id}}", + "This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts.": "This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts." } } diff --git a/web/default/src/i18n/locales/vi.json b/web/default/src/i18n/locales/vi.json index e6ba04fe..37bf3d27 100644 --- a/web/default/src/i18n/locales/vi.json +++ b/web/default/src/i18n/locales/vi.json @@ -5746,6 +5746,85 @@ "Enter exact quotas ({{count}})_other": "Nhập các hạn mức chính xác ({{count}} tác vụ)", "Enter an exact final quota for each selected task. Each value is checked against its own reserved quota.": "Nhập hạn mức cuối chính xác cho từng tác vụ đã chọn. Mỗi giá trị sẽ được kiểm tra theo hạn mức đã giữ riêng của tác vụ đó.", "Apply exact settlements": "Áp dụng các quyết toán chính xác", - "Allowed range: 0 to {{quota}} ({{quotaPerUnit}} quota = $1).": "Phạm vi cho phép: từ 0 đến {{quota}} ({{quotaPerUnit}} đơn vị hạn mức = $1)." + "Allowed range: 0 to {{quota}} ({{quotaPerUnit}} quota = $1).": "Phạm vi cho phép: từ 0 đến {{quota}} ({{quotaPerUnit}} đơn vị hạn mức = $1).", + "Automatic recovery mode": "Automatic recovery mode", + "Probe then automatic": "Probe then automatic", + "Manual recovery": "Manual recovery", + "Timeout count scope": "Timeout count scope", + "Same request mode": "Same request mode", + "Combined modes": "Combined modes", + "Count streaming and non-streaming timeouts separately or together.": "Count streaming and non-streaming timeouts separately or together.", + "Probe mode recovers after configured successful probes; manual mode requires an administrator.": "Probe mode recovers after configured successful probes; manual mode requires an administrator.", + "Channel recovery failed": "Channel recovery failed", + "Channel recovery requested": "Channel recovery requested", + "Recover channel": "Recover channel", + "We could not load the channel health policy.": "We could not load the channel health policy.", + "Critical": "Critical", + "Channel timeout notifications": "Channel timeout notifications", + "Delivery status": "Delivery status", + "Recovery mode": "Recovery mode", + "Automatic": "Automatic", + "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.", + "Extend penalty on repeated timeout": "Extend penalty on repeated timeout", + "Keep the current penalty deadline when repeated timeouts continue during an active penalty.": "Keep the current penalty deadline when repeated timeouts continue during an active penalty.", + "{{count}} priority point": "{{count}} priority point", + "{{count}} priority points": "{{count}} priority points", + "{{count}} timeout": "{{count}} timeout", + "{{count}} timeouts": "{{count}} timeouts", + "Streaming first-result timeout must be between 0 and 86400 seconds.": "Streaming first-result timeout must be between 0 and 86400 seconds.", + "Non-streaming response timeout must be between 0 and 86400 seconds.": "Non-streaming response timeout must be between 0 and 86400 seconds.", + "Priority deduction must be a non-negative integer.": "Priority deduction must be a non-negative integer.", + "Disable count must be at least 2.": "Disable count must be at least 2.", + "Disable window must be between 60 and 2592000 seconds.": "Disable window must be between 60 and 2592000 seconds.", + "Minimum samples must be a non-negative integer.": "Minimum samples must be a non-negative integer.", + "Timeout ratio must be between 0 and 100 percent.": "Timeout ratio must be between 0 and 100 percent.", + "Disable duration must be between 0 and 604800 seconds.": "Disable duration must be between 0 and 604800 seconds.", + "Successful probe count must be between 1 and 10.": "Successful probe count must be between 1 and 10.", + "Penalty cooldown must be between 0 and 2592000 seconds.": "Penalty cooldown must be between 0 and 2592000 seconds.", + "Alert repeat interval must be between 0 and 2592000 seconds.": "Alert repeat interval must be between 0 and 2592000 seconds.", + "Recovery mode is invalid.": "Recovery mode is invalid.", + "Timeout count scope is invalid.": "Timeout count scope is invalid.", + "Channel health": "Channel health", + "Channel timeout governance": "Channel timeout governance", + "This view combines timeout alerts with the existing monitor_setting policy. Values are stored through the existing system option endpoint; no new database table or field is used.": "This view combines timeout alerts with the existing monitor_setting policy. Values are stored through the existing system option endpoint; no new database table or field is used.", + "Channel timeout alerts": "Channel timeout alerts", + "Priority demotions": "Priority demotions", + "Automatic disable alerts": "Automatic disable alerts", + "Timeout policy": "Timeout policy", + "Super administrators can change the existing monitor_setting values.": "Super administrators can change the existing monitor_setting values.", + "Only super administrators can change these values.": "Only super administrators can change these values.", + "Enable automatic priority demotion": "Enable automatic priority demotion", + "Reduce a channel priority after an eligible upstream timeout.": "Reduce a channel priority after an eligible upstream timeout.", + "Enable automatic channel disable": "Enable automatic channel disable", + "Disable a channel when its configured timeout count and ratio gates are met.": "Disable a channel when its configured timeout count and ratio gates are met.", + "Streaming first-result timeout": "Streaming first-result timeout", + "Seconds; 0 disables this threshold.": "Seconds; 0 disables this threshold.", + "Non-streaming response timeout": "Non-streaming response timeout", + "Priority deduction": "Priority deduction", + "Administrator-defined priority points per timeout.": "Administrator-defined priority points per timeout.", + "Penalty cooldown": "Penalty cooldown", + "Seconds before the same channel can be penalized again.": "Seconds before the same channel can be penalized again.", + "Alert repeat interval": "Alert repeat interval", + "Seconds between repeated notifications for one channel.": "Seconds between repeated notifications for one channel.", + "Disable after timeout count": "Disable after timeout count", + "Consecutive/window count used by the existing auto-disable policy.": "Consecutive/window count used by the existing auto-disable policy.", + "Auto-disable window": "Auto-disable window", + "Seconds used to evaluate timeout counts.": "Seconds used to evaluate timeout counts.", + "Minimum timeout samples": "Minimum timeout samples", + "Minimum observations before ratio-based disable is eligible.": "Minimum observations before ratio-based disable is eligible.", + "Timeout ratio threshold": "Timeout ratio threshold", + "Percentage threshold for the existing auto-disable policy.": "Percentage threshold for the existing auto-disable policy.", + "Auto-disable duration": "Auto-disable duration", + "Seconds a channel remains disabled before recovery checks.": "Seconds a channel remains disabled before recovery checks.", + "Successful recovery probes": "Successful recovery probes", + "Successful probes required before automatic recovery.": "Successful probes required before automatic recovery.", + "Expose channel demotion and disable transitions to the SmartOps alert stream.": "Expose channel demotion and disable transitions to the SmartOps alert stream.", + "Save timeout policy": "Save timeout policy", + "Channel alert stream": "Channel alert stream", + "Live channel timeout transitions from the existing SmartOps alert endpoint.": "Live channel timeout transitions from the existing SmartOps alert endpoint.", + "No active channel timeout alerts.": "No active channel timeout alerts.", + "Unknown channel": "Unknown channel", + "Channel {{id}}": "Channel {{id}}", + "This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts.": "This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts." } } diff --git a/web/default/src/i18n/locales/zh.json b/web/default/src/i18n/locales/zh.json index 6dccbeb7..bebaaa61 100644 --- a/web/default/src/i18n/locales/zh.json +++ b/web/default/src/i18n/locales/zh.json @@ -5493,7 +5493,6 @@ "Zhipu": "智谱", "Zhipu V4": "智谱 V4", "Zoom": "缩放", - "Smart Operations": "智能运维", "Smart Operations Center": "智能运维中心", "Production performance": "生产性能", "Production performance is not loaded automatically": "生产性能不会自动加载", @@ -5585,6 +5584,59 @@ "Throughput data is partial": "吞吐量数据不完整", "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete.": "当前显示已存储和本地性能样本，但无法读取共享活动桶；最近的多节点吞吐量可能不完整。", "Active Alerts": "活动告警", + "Channel health": "渠道健康", + "Smart Operations": "智能运维", + "Channel timeout governance": "渠道超时治理", + "This view combines timeout alerts with the existing monitor_setting policy. Values are stored through the existing system option endpoint; no new database table or field is used.": "此页面将超时告警与现有 monitor_setting 策略合并展示。配置通过现有系统选项接口保存，不新增数据库表或字段。", + "Channel timeout alerts": "渠道超时告警", + "Priority demotions": "优先级降级", + "Automatic disable alerts": "自动禁用告警", + "Notifications": "通知", + "Timeout policy": "超时策略", + "Super administrators can change the existing monitor_setting values.": "超级管理员可以修改现有 monitor_setting 配置。", + "Only super administrators can change these values.": "只有超级管理员可以修改这些配置。", + "Enable automatic priority demotion": "启用自动优先级降级", + "Enable automatic channel disable": "启用自动渠道禁用", + "Disable a channel when its configured timeout count and ratio gates are met.": "达到配置的超时次数和比例条件后自动禁用渠道。", + "Reduce a channel priority after an eligible upstream timeout.": "上游请求发生符合条件的超时后降低渠道优先级。", + "Streaming first-result timeout": "流式首字节超时", + "Non-streaming response timeout": "非流式响应超时", + "Seconds; 0 disables this threshold.": "单位为秒；设置为 0 时关闭该阈值。", + "Priority deduction": "优先级扣减值", + "Administrator-defined priority points per timeout.": "每次超时时扣减的管理员指定优先级数值。", + "Penalty cooldown": "惩罚冷却时间", + "Seconds before the same channel can be penalized again.": "同一渠道再次受到惩罚前等待的秒数。", + "Alert repeat interval": "告警重复间隔", + "Seconds between repeated notifications for one channel.": "同一渠道重复通知之间的秒数。", + "Disable after timeout count": "超时多少次后禁用", + "Consecutive/window count used by the existing auto-disable policy.": "现有自动禁用策略使用的连续或窗口计数。", + "Auto-disable window": "自动禁用统计窗口", + "Seconds used to evaluate timeout counts.": "用于统计超时次数的秒数。", + "Minimum timeout samples": "最少超时样本数", + "Minimum observations before ratio-based disable is eligible.": "达到该观测数后才允许按比例自动禁用。", + "Timeout ratio threshold": "超时比例阈值", + "Percentage threshold for the existing auto-disable policy.": "现有自动禁用策略使用的百分比阈值。", + "Auto-disable duration": "自动禁用时长", + "Seconds a channel remains disabled before recovery checks.": "渠道保持禁用并等待恢复检查的秒数。", + "Successful recovery probes": "成功恢复探测次数", + "Successful probes required before automatic recovery.": "自动恢复前要求的成功探测次数。", + "Channel timeout notifications": "渠道超时通知", + "Delivery status": "投递状态", + "Recovery mode": "恢复模式", + "Automatic": "自动", + "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "自动恢复会在冷却时间后恢复惩罚；手动恢复会保留惩罚，直到管理员操作。", + "Extend penalty on repeated timeout": "重复超时时延长惩罚", + "Keep the current penalty deadline when repeated timeouts continue during an active penalty.": "惩罚生效期间再次超时时，是否保留并延长当前惩罚截止时间。", + "Expose channel demotion and disable transitions to the SmartOps alert stream.": "将渠道降级和禁用状态变化发布到 SmartOps 告警流。", + "Save timeout policy": "保存超时策略", + "Channel alert stream": "渠道告警流", + "Live channel timeout transitions from the existing SmartOps alert endpoint.": "来自现有 SmartOps 告警接口的实时渠道超时状态变化。", + "No active channel timeout alerts.": "当前没有活动的渠道超时告警。", + "Critical": "严重", + "Unknown channel": "未知渠道", + "Channel {{id}}": "渠道 {{id}}", + "Channel priority demotion": "渠道优先级降级", + "Channel automatically disabled": "渠道已自动禁用", "Alerts requiring administrator attention": "需要管理员关注的告警", "SmartOps reports sustained host pressure here for manual investigation and repair.": "SmartOps 会在此报告持续的主机资源压力，供管理员人工排查和修复。", "Resource": "资源", @@ -5597,6 +5649,7 @@ "We could not load active alerts.": "无法加载活动告警。", "No active alerts.": "当前没有活动告警。", "This process is not currently reporting a sustained CPU, memory, or disk alert.": "当前进程暂未报告持续的 CPU、内存或磁盘告警。", + "This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts.": "当前进程暂未报告持续的主机压力、渠道超时或账务对账告警。", "Add a password or sign in again with a linked account to continue.": "添加密码，或使用已关联账户重新登录以继续。", "Add a password or sign in again with a linked account to unlock sensitive operations.": "添加密码，或使用已关联账户重新登录，以解锁敏感操作。", "Additional verification required": "需要额外验证", @@ -5746,6 +5799,35 @@ "Enter exact quotas ({{count}})_other": "输入精确额度（{{count}} 个）", "Enter an exact final quota for each selected task. Each value is checked against its own reserved quota.": "为每个所选任务输入精确最终额度。每个数值都会按照该任务自身的预留额度校验。", "Apply exact settlements": "应用精确结算", - "Allowed range: 0 to {{quota}} ({{quotaPerUnit}} quota = $1).": "允许范围：0 至 {{quota}}（{{quotaPerUnit}} 额度 = $1）。" + "Allowed range: 0 to {{quota}} ({{quotaPerUnit}} quota = $1).": "允许范围：0 至 {{quota}}（{{quotaPerUnit}} 额度 = $1）。", + "Automatic recovery mode": "自动恢复模式", + "Probe then automatic": "探测成功后自动恢复", + "Manual recovery": "手动恢复", + "Timeout count scope": "超时计数范围", + "Same request mode": "按请求模式分别计数", + "Combined modes": "合并流式与非流式", + "Count streaming and non-streaming timeouts separately or together.": "分别或合并统计流式与非流式超时。", + "Probe mode recovers after configured successful probes; manual mode requires an administrator.": "探测模式在达到配置的成功次数后恢复；手动模式需要管理员操作。", + "Channel recovery failed": "渠道恢复失败", + "Channel recovery requested": "已请求渠道恢复", + "Recover channel": "恢复渠道", + "We could not load the channel health policy.": "无法加载渠道健康策略。", + "{{count}} priority point": "{{count}} 个优先级点", + "{{count}} priority points": "{{count}} 个优先级点", + "{{count}} timeout": "{{count}} 次超时", + "{{count}} timeouts": "{{count}} 次超时", + "Streaming first-result timeout must be between 0 and 86400 seconds.": "流式首结果超时必须在 0 到 86400 秒之间。", + "Non-streaming response timeout must be between 0 and 86400 seconds.": "非流式响应超时必须在 0 到 86400 秒之间。", + "Priority deduction must be a non-negative integer.": "优先级扣减必须是非负整数。", + "Disable count must be at least 2.": "禁用次数至少为 2。", + "Disable window must be between 60 and 2592000 seconds.": "禁用统计窗口必须在 60 到 2592000 秒之间。", + "Minimum samples must be a non-negative integer.": "最小样本数必须是非负整数。", + "Timeout ratio must be between 0 and 100 percent.": "超时比例必须在 0 到 100% 之间。", + "Disable duration must be between 0 and 604800 seconds.": "禁用时长必须在 0 到 604800 秒之间。", + "Successful probe count must be between 1 and 10.": "成功探测次数必须在 1 到 10 之间。", + "Penalty cooldown must be between 0 and 2592000 seconds.": "惩罚冷却时间必须在 0 到 2592000 秒之间。", + "Alert repeat interval must be between 0 and 2592000 seconds.": "告警重复间隔必须在 0 到 2592000 秒之间。", + "Recovery mode is invalid.": "恢复模式无效。", + "Timeout count scope is invalid.": "超时计数范围无效。" } } diff --git a/web/default/src/routeTree.gen.ts b/web/default/src/routeTree.gen.ts index 5092b979..28db292f 100644 --- a/web/default/src/routeTree.gen.ts +++ b/web/default/src/routeTree.gen.ts @@ -54,6 +54,7 @@ import { Route as AuthenticatedSmartOpsSystemInfoRouteImport } from './routes/_a import { Route as AuthenticatedSmartOpsProductionPerformanceRouteImport } from './routes/_authenticated/smart-ops/production-performance' import { Route as AuthenticatedSmartOpsModelPerformanceRouteImport } from './routes/_authenticated/smart-ops/model-performance' import { Route as AuthenticatedSmartOpsChannelPerformanceRouteImport } from './routes/_authenticated/smart-ops/channel-performance' +import { Route as AuthenticatedSmartOpsChannelHealthRouteImport } from './routes/_authenticated/smart-ops/channel-health' import { Route as AuthenticatedSmartOpsAlertsRouteImport } from './routes/_authenticated/smart-ops/alerts' import { Route as AuthenticatedModelsSectionRouteImport } from './routes/_authenticated/models/$section' import { Route as AuthenticatedErrorsErrorRouteImport } from './routes/_authenticated/errors/$error' @@ -315,6 +316,12 @@ const AuthenticatedSmartOpsChannelPerformanceRoute = path: '/smart-ops/channel-performance', getParentRoute: () => AuthenticatedRouteRoute, } as any) +const AuthenticatedSmartOpsChannelHealthRoute = + AuthenticatedSmartOpsChannelHealthRouteImport.update({ + id: '/smart-ops/channel-health', + path: '/smart-ops/channel-health', + getParentRoute: () => AuthenticatedRouteRoute, + } as any) const AuthenticatedSmartOpsAlertsRoute = AuthenticatedSmartOpsAlertsRouteImport.update({ id: '/smart-ops/alerts', @@ -465,6 +472,7 @@ export interface FileRoutesByFullPath { '/errors/$error': typeof AuthenticatedErrorsErrorRoute '/models/$section': typeof AuthenticatedModelsSectionRoute '/smart-ops/alerts': typeof AuthenticatedSmartOpsAlertsRoute + '/smart-ops/channel-health': typeof AuthenticatedSmartOpsChannelHealthRoute '/smart-ops/channel-performance': typeof AuthenticatedSmartOpsChannelPerformanceRoute '/smart-ops/model-performance': typeof AuthenticatedSmartOpsModelPerformanceRoute '/smart-ops/production-performance': typeof AuthenticatedSmartOpsProductionPerformanceRoute @@ -529,6 +537,7 @@ export interface FileRoutesByTo { '/errors/$error': typeof AuthenticatedErrorsErrorRoute '/models/$section': typeof AuthenticatedModelsSectionRoute '/smart-ops/alerts': typeof AuthenticatedSmartOpsAlertsRoute + '/smart-ops/channel-health': typeof AuthenticatedSmartOpsChannelHealthRoute '/smart-ops/channel-performance': typeof AuthenticatedSmartOpsChannelPerformanceRoute '/smart-ops/model-performance': typeof AuthenticatedSmartOpsModelPerformanceRoute '/smart-ops/production-performance': typeof AuthenticatedSmartOpsProductionPerformanceRoute @@ -597,6 +606,7 @@ export interface FileRoutesById { '/_authenticated/errors/$error': typeof AuthenticatedErrorsErrorRoute '/_authenticated/models/$section': typeof AuthenticatedModelsSectionRoute '/_authenticated/smart-ops/alerts': typeof AuthenticatedSmartOpsAlertsRoute + '/_authenticated/smart-ops/channel-health': typeof AuthenticatedSmartOpsChannelHealthRoute '/_authenticated/smart-ops/channel-performance': typeof AuthenticatedSmartOpsChannelPerformanceRoute '/_authenticated/smart-ops/model-performance': typeof AuthenticatedSmartOpsModelPerformanceRoute '/_authenticated/smart-ops/production-performance': typeof AuthenticatedSmartOpsProductionPerformanceRoute @@ -664,6 +674,7 @@ export interface FileRouteTypes { | '/errors/$error' | '/models/$section' | '/smart-ops/alerts' + | '/smart-ops/channel-health' | '/smart-ops/channel-performance' | '/smart-ops/model-performance' | '/smart-ops/production-performance' @@ -728,6 +739,7 @@ export interface FileRouteTypes { | '/errors/$error' | '/models/$section' | '/smart-ops/alerts' + | '/smart-ops/channel-health' | '/smart-ops/channel-performance' | '/smart-ops/model-performance' | '/smart-ops/production-performance' @@ -795,6 +807,7 @@ export interface FileRouteTypes { | '/_authenticated/errors/$error' | '/_authenticated/models/$section' | '/_authenticated/smart-ops/alerts' + | '/_authenticated/smart-ops/channel-health' | '/_authenticated/smart-ops/channel-performance' | '/_authenticated/smart-ops/model-performance' | '/_authenticated/smart-ops/production-performance' @@ -1168,6 +1181,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof AuthenticatedSmartOpsChannelPerformanceRouteImport parentRoute: typeof AuthenticatedRouteRoute } + '/_authenticated/smart-ops/channel-health': { + id: '/_authenticated/smart-ops/channel-health' + path: '/smart-ops/channel-health' + fullPath: '/smart-ops/channel-health' + preLoaderRoute: typeof AuthenticatedSmartOpsChannelHealthRouteImport + parentRoute: typeof AuthenticatedRouteRoute + } '/_authenticated/smart-ops/alerts': { id: '/_authenticated/smart-ops/alerts' path: '/smart-ops/alerts' @@ -1402,6 +1422,7 @@ interface AuthenticatedRouteRouteChildren { AuthenticatedErrorsErrorRoute: typeof AuthenticatedErrorsErrorRoute AuthenticatedModelsSectionRoute: typeof AuthenticatedModelsSectionRoute AuthenticatedSmartOpsAlertsRoute: typeof AuthenticatedSmartOpsAlertsRoute + AuthenticatedSmartOpsChannelHealthRoute: typeof AuthenticatedSmartOpsChannelHealthRoute AuthenticatedSmartOpsChannelPerformanceRoute: typeof AuthenticatedSmartOpsChannelPerformanceRoute AuthenticatedSmartOpsModelPerformanceRoute: typeof AuthenticatedSmartOpsModelPerformanceRoute AuthenticatedSmartOpsProductionPerformanceRoute: typeof AuthenticatedSmartOpsProductionPerformanceRoute @@ -1430,6 +1451,8 @@ const AuthenticatedRouteRouteChildren: AuthenticatedRouteRouteChildren = { AuthenticatedErrorsErrorRoute: AuthenticatedErrorsErrorRoute, AuthenticatedModelsSectionRoute: AuthenticatedModelsSectionRoute, AuthenticatedSmartOpsAlertsRoute: AuthenticatedSmartOpsAlertsRoute, + AuthenticatedSmartOpsChannelHealthRoute: + AuthenticatedSmartOpsChannelHealthRoute, AuthenticatedSmartOpsChannelPerformanceRoute: AuthenticatedSmartOpsChannelPerformanceRoute, AuthenticatedSmartOpsModelPerformanceRoute: diff --git a/web/default/src/routes/_authenticated/smart-ops/channel-health.tsx b/web/default/src/routes/_authenticated/smart-ops/channel-health.tsx new file mode 100644 index 00000000..f0a94b68 --- /dev/null +++ b/web/default/src/routes/_authenticated/smart-ops/channel-health.tsx @@ -0,0 +1,34 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import { createFileRoute, redirect } from '@tanstack/react-router' +import { useAuthStore } from '@/stores/auth-store' +import { ROLE } from '@/lib/roles' +import { ChannelHealth } from '@/features/smart-ops/channel-health' + +export const Route = createFileRoute( + '/_authenticated/smart-ops/channel-health' +)({ + beforeLoad: () => { + const { auth } = useAuthStore.getState() + if (!auth.user || auth.user.role < ROLE.ADMIN) { + throw redirect({ to: '/403' }) + } + }, + component: ChannelHealth, +}) From 7cb6a7a515486a60aeedee7ad2236d848b5f1f2e Mon Sep 17 00:00:00 2001 From: CSCITech Date: Fri, 2 Oct 2026 22:50:07 +0800 Subject: [PATCH 02/12] fix: close channel health review findings --- controller/smart_ops_alert.go | 2 +- model/channel_health_routing.go | 71 ++++++++++- pkg/channelhealth/runtime_test.go | 1 - .../common/channel_health_observation_test.go | 17 +++ relay/common/relay_info.go | 115 +++++++++++++----- relay/helper/stream_scanner.go | 5 +- setting/operation_setting/monitor_setting.go | 6 +- .../smart-ops/channel-health/index.tsx | 44 +++---- 8 files changed, 199 insertions(+), 62 deletions(-) diff --git a/controller/smart_ops_alert.go b/controller/smart_ops_alert.go index db08365a..4c6076ec 100644 --- a/controller/smart_ops_alert.go +++ b/controller/smart_ops_alert.go @@ -104,7 +104,7 @@ func UpdateChannelHealthPolicy(c *gin.Context) { return } recordManageAudit(c, "smart_ops.channel_health_policy_update", map[string]interface{}{ - "keys": request.Updates, + "updates": values, }) c.JSON(http.StatusOK, gin.H{ "success": true, diff --git a/model/channel_health_routing.go b/model/channel_health_routing.go index 7831917d..167e7a83 100644 --- a/model/channel_health_routing.go +++ b/model/channel_health_routing.go @@ -2,12 +2,32 @@ package model import ( "context" + "sync" "time" + maxcommon "github.com/MAX-API-Next/MAX-API/common" "github.com/MAX-API-Next/MAX-API/pkg/channelhealth" "github.com/MAX-API-Next/MAX-API/setting/operation_setting" + "github.com/go-redis/redis/v8" ) +const ( + runtimeStateCacheTTL = time.Second + runtimeStateFailureCooldown = 250 * time.Millisecond +) + +type runtimeStateCacheEntry struct { + client *redis.Client + state channelhealth.RuntimeState + expiresAt time.Time + retryAfter time.Time +} + +var runtimeStateCache = struct { + sync.Mutex + entries map[int]runtimeStateCacheEntry +}{entries: make(map[int]runtimeStateCacheEntry)} + type channelRoutingCandidate struct { channel *Channel priority int64 @@ -22,14 +42,59 @@ func loadChannelRuntimeStates(channelIDs []int) map[int]channelhealth.RuntimeSta if !channelHealthRoutingEnabled() || len(channelIDs) == 0 { return nil } + now := time.Now() + client := maxcommon.RDB + if !maxcommon.RedisEnabled || client == nil { + return nil + } + states := make(map[int]channelhealth.RuntimeState, len(channelIDs)) + pending := make([]int, 0, len(channelIDs)) + runtimeStateCache.Lock() + for _, channelID := range channelIDs { + entry, ok := runtimeStateCache.entries[channelID] + if ok && entry.client == client && now.Before(entry.retryAfter) { + continue + } + if ok && entry.client == client && now.Before(entry.expiresAt) { + states[channelID] = entry.state + continue + } + pending = append(pending, channelID) + } + runtimeStateCache.Unlock() + if len(pending) == 0 { + return states + } ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) defer cancel() - states, err := channelhealth.LoadRuntimeStates(ctx, channelIDs) + loaded, err := channelhealth.LoadRuntimeStates(ctx, pending) if err != nil { + runtimeStateCache.Lock() + for _, channelID := range pending { + runtimeStateCache.entries[channelID] = runtimeStateCacheEntry{ + client: client, + retryAfter: now.Add(runtimeStateFailureCooldown), + } + } + runtimeStateCache.Unlock() // Redis is an optional runtime dependency for this feature. The caller - // must keep the existing base-priority selection when it is unavailable. - return nil + // must keep the existing base-priority selection for channels without a + // cached state. Freshly cached states remain usable during the cooldown. + if len(states) == 0 { + return nil + } + return states + } + runtimeStateCache.Lock() + for channelID, state := range loaded { + runtimeStateCache.entries[channelID] = runtimeStateCacheEntry{ + client: client, + state: state, + expiresAt: now.Add(runtimeStateCacheTTL), + } + states[channelID] = state } + runtimeStateCache.Unlock() return states } diff --git a/pkg/channelhealth/runtime_test.go b/pkg/channelhealth/runtime_test.go index 9adbac9f..78044118 100644 --- a/pkg/channelhealth/runtime_test.go +++ b/pkg/channelhealth/runtime_test.go @@ -87,7 +87,6 @@ func TestEvaluateTimeoutGuardRetriesConcurrentRedisTransactions(t *testing.T) { var waitGroup sync.WaitGroup waitGroup.Add(attempts) for index := 0; index < attempts; index++ { - index := index go func() { defer waitGroup.Done() _, err := EvaluateTimeoutGuard(context.Background(), AttemptEvidence{ diff --git a/relay/common/channel_health_observation_test.go b/relay/common/channel_health_observation_test.go index f321f623..22960b28 100644 --- a/relay/common/channel_health_observation_test.go +++ b/relay/common/channel_health_observation_test.go @@ -38,3 +38,20 @@ func TestRelayInfoRecordsOneFirstResponsePerChannelAttempt(t *testing.T) { require.Equal(t, 2, observations[0].RetryIndex) require.True(t, observations[0].AttemptLatencyMS >= 0) } + +func TestRelayInfoFirstResponseEvaluationHasBoundedGrace(t *testing.T) { + info := &RelayInfo{} + finish := info.BeginFirstResponseEvaluation() + + require.False(t, info.FirstResponseDeadlineExpired()) + info.SetFirstResponseTime() + require.False(t, info.FirstResponseDeadlineExpired()) + finish() + require.False(t, info.FirstResponseDeadlineExpired()) + + pending := &RelayInfo{} + finishPending := pending.BeginFirstResponseEvaluation() + t.Cleanup(finishPending) + time.Sleep(firstResponseEvaluationGrace + 10*time.Millisecond) + require.True(t, pending.FirstResponseDeadlineExpired()) +} diff --git a/relay/common/relay_info.go b/relay/common/relay_info.go index c7f55930..1a0b6f83 100644 --- a/relay/common/relay_info.go +++ b/relay/common/relay_info.go @@ -7,6 +7,7 @@ import ( "fmt" "strconv" "strings" + "sync" "sync/atomic" "time" @@ -91,19 +92,22 @@ type TokenCountMeta struct { } type RelayInfo struct { - TokenId int - TokenKey string - TokenGroup string - UserId int - UsingGroup string // 使用的分组，当auto跨分组重试时，会变动 - UserGroup string // 用户所在分组 - TokenUnlimited bool - StartTime time.Time - FirstResponseTime time.Time - isFirstResponse bool - channelAttemptStartTime time.Time - channelFirstResponseRecorded atomic.Bool - channelFirstResponseSignal chan struct{} + TokenId int + TokenKey string + TokenGroup string + UserId int + UsingGroup string // 使用的分组，当auto跨分组重试时，会变动 + UserGroup string // 用户所在分组 + TokenUnlimited bool + StartTime time.Time + FirstResponseTime time.Time + isFirstResponse bool + channelAttemptStartTime time.Time + channelFirstResponseMu sync.Mutex + channelFirstResponseEvalAt time.Time + channelFirstResponseEvaluating bool + channelFirstResponseRecorded atomic.Bool + channelFirstResponseSignal chan struct{} //SendLastReasoningResponse bool IsStream bool IsGeminiBatchEmbedding bool @@ -761,6 +765,12 @@ func (info *RelayInfo) SetFirstResponseTime() { if info == nil { return } + info.channelFirstResponseMu.Lock() + defer info.channelFirstResponseMu.Unlock() + info.setFirstResponseTimeLocked() +} + +func (info *RelayInfo) setFirstResponseTimeLocked() { if info.isFirstResponse { info.FirstResponseTime = time.Now() info.isFirstResponse = false @@ -775,6 +785,46 @@ func (info *RelayInfo) SetFirstResponseTime() { info.recordChannelHealthSuccess() } +const firstResponseEvaluationGrace = 100 * time.Millisecond + +// BeginFirstResponseEvaluation marks the short window in which a stream +// handler is deciding whether the current payload is a deliverable first +// response. The watchdog gives that decision a bounded grace period instead of +// waiting on the response write itself. +func (info *RelayInfo) BeginFirstResponseEvaluation() func() { + if info == nil { + return func() {} + } + info.channelFirstResponseMu.Lock() + if !info.channelFirstResponseRecorded.Load() { + info.channelFirstResponseEvaluating = true + info.channelFirstResponseEvalAt = time.Now() + } + info.channelFirstResponseMu.Unlock() + return func() { + info.channelFirstResponseMu.Lock() + info.channelFirstResponseEvaluating = false + info.channelFirstResponseEvalAt = time.Time{} + info.channelFirstResponseMu.Unlock() + } +} + +func (info *RelayInfo) FirstResponseDeadlineExpired() bool { + if info == nil { + return true + } + info.channelFirstResponseMu.Lock() + defer info.channelFirstResponseMu.Unlock() + if info.channelFirstResponseRecorded.Load() { + return false + } + if info.channelFirstResponseEvaluating && + time.Since(info.channelFirstResponseEvalAt) < firstResponseEvaluationGrace { + return false + } + return true +} + // FirstResponseSignal is closed once the current channel attempt has produced // a first valid, deliverable result. It is used by the stream watchdog without // exposing the internal deduplication state to relay callers. @@ -884,31 +934,32 @@ func (info *RelayInfo) RecordChannelTimeout(event string) { return } setting := operation_setting.GetMonitorSetting() - if setting == nil { + if setting == nil || (!setting.AutoPriorityDemotionEnabled && !setting.TimeoutAutoDisableEnabled) { return } requestMode := channelhealth.RequestModeNonStreaming if info.IsStream { requestMode = channelhealth.RequestModeStreaming } - decisionContext, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) - defer cancel() - result, err := channelhealth.EvaluateTimeoutGuard( - decisionContext, - channelhealth.AttemptEvidence{ - ChannelID: info.ChannelId, - AttemptID: info.channelHealthAttemptID(), - RequestMode: requestMode, - TimeoutKind: timeoutKind, - TimeoutEligible: true, - AutoBan: info.ChannelAutoBan, - ObservedAt: time.Now(), - }, - setting, - ) - if err == nil && result.Transition != "" { + evidence := channelhealth.AttemptEvidence{ + ChannelID: info.ChannelId, + AttemptID: info.channelHealthAttemptID(), + RequestMode: requestMode, + TimeoutKind: timeoutKind, + TimeoutEligible: true, + AutoBan: info.ChannelAutoBan, + ObservedAt: time.Now(), + } + settingSnapshot := *setting + go func(evidence channelhealth.AttemptEvidence, setting operation_setting.MonitorSetting) { + decisionContext, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + result, err := channelhealth.EvaluateTimeoutGuard(decisionContext, evidence, &setting) + if err != nil || result.Transition == "" { + return + } common.PublishChannelHealthRuntimeTransition(common.ChannelHealthRuntimeTransition{ - ChannelID: info.ChannelId, + ChannelID: evidence.ChannelID, Transition: result.Transition, Penalty: result.State.Penalty, PenaltyUntil: result.State.PenaltyUntil, @@ -916,7 +967,7 @@ func (info *RelayInfo) RecordChannelTimeout(event string) { DisabledUntil: result.State.DisabledUntil, ObservedAt: result.State.LastObservedAt, }) - } + }(evidence, settingSnapshot) } func (info *RelayInfo) HasSendResponse() bool { diff --git a/relay/helper/stream_scanner.go b/relay/helper/stream_scanner.go index 8d41d27c..8c8b66be 100644 --- a/relay/helper/stream_scanner.go +++ b/relay/helper/stream_scanner.go @@ -230,7 +230,9 @@ func StreamScannerHandler(c *gin.Context, resp *http.Response, info *relaycommon sr := newStreamResult(info.StreamStatus) for data := range dataChan { sr.reset() + finishFirstResponseEvaluation := info.BeginFirstResponseEvaluation() func() { + defer finishFirstResponseEvaluation() writeMutex.Lock() defer writeMutex.Unlock() ExtendWriteDeadline(c) @@ -331,11 +333,12 @@ waitLoop: continue case <-firstResultTimeoutChan: firstResultTimeoutChan = nil - if !info.HasRecordedChannelFirstResponse() { + if info.FirstResponseDeadlineExpired() { info.StreamStatus.SetEndReason(relaycommon.StreamEndReasonTimeout, nil) info.RecordChannelTimeout("streaming_first_result_timeout") break waitLoop } + firstResultTimeoutChan = time.After(100 * time.Millisecond) continue case <-timeoutChan: info.StreamStatus.SetEndReason(relaycommon.StreamEndReasonTimeout, nil) diff --git a/setting/operation_setting/monitor_setting.go b/setting/operation_setting/monitor_setting.go index 7c2bee3c..7b052a50 100644 --- a/setting/operation_setting/monitor_setting.go +++ b/setting/operation_setting/monitor_setting.go @@ -164,10 +164,10 @@ func ValidateMonitorSettingOption(key, value string) error { return nil } -func validateMonitorInt(key, value string, min, max int) error { +func validateMonitorInt(key, value string, lo, hi int) error { parsed, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64) - if err != nil || parsed < int64(min) || parsed > int64(max) { - return fmt.Errorf("%s must be an integer between %d and %d", key, min, max) + if err != nil || parsed < int64(lo) || parsed > int64(hi) { + return fmt.Errorf("%s must be an integer between %d and %d", key, lo, hi) } return nil } diff --git a/web/default/src/features/smart-ops/channel-health/index.tsx b/web/default/src/features/smart-ops/channel-health/index.tsx index 0ca04739..ba64b465 100644 --- a/web/default/src/features/smart-ops/channel-health/index.tsx +++ b/web/default/src/features/smart-ops/channel-health/index.tsx @@ -395,11 +395,15 @@ export function ChannelHealth(): ReactElement { retry: false, staleTime: 30 * 1000, }) + const [draftOverride, setDraftOverride] = useState( + null + ) const updatePolicy = useMutation({ mutationFn: updateChannelHealthPolicy, onSuccess: (response) => { if (response.success) { toast.success(t('Setting updated successfully')) + setDraftOverride(null) void policyQuery.refetch() } else { toast.error(response.message || t('Failed to update setting')) @@ -427,9 +431,6 @@ export function ChannelHealth(): ReactElement { () => parseDraft(policyQuery.data?.data), [policyQuery.data?.data] ) - const [draftOverride, setDraftOverride] = useState( - null - ) const draft = draftOverride ?? loadedDraft const updateDraft = ( updater: (current: ChannelHealthDraft) => ChannelHealthDraft @@ -954,24 +955,25 @@ export function ChannelHealth(): ReactElement { ? '—' : dateFormatter.format(new Date(observedAt))} - {canEdit && ( - - )} + {canEdit && + /^\d+$/.test(channelIdFromAlert(alert)) && ( + + )} ) })} From f0b8fd406f06c1893a9a6e10a81814aa7b5f1ee7 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Sat, 3 Oct 2026 00:22:01 +0800 Subject: [PATCH 03/12] fix: harden channel health recovery and routing --- controller/channel-test.go | 57 +++++++++++--- controller/channel_health_recovery_test.go | 4 + model/channel_health_routing.go | 11 ++- model/channel_health_routing_test.go | 21 ++++++ pkg/channelhealth/runtime.go | 70 +++++++++++++++--- pkg/channelhealth/runtime_test.go | 74 +++++++++++++++++++ .../smart-ops/channel-health/index.tsx | 2 +- 7 files changed, 215 insertions(+), 24 deletions(-) diff --git a/controller/channel-test.go b/controller/channel-test.go index cf86324a..68508e82 100644 --- a/controller/channel-test.go +++ b/controller/channel-test.go @@ -959,7 +959,7 @@ func TestChannel(c *gin.Context) { var testAllChannelsLock sync.Mutex var testAllChannelsRunning bool = false -func testChannels(channels []*model.Channel, testUserID int, notify bool, allowDisable bool) error { +func testChannels(channels []*model.Channel, testUserID int, notify bool, allowDisable bool, runtimeStates map[int]channelhealth.RuntimeState) error { testAllChannelsLock.Lock() if testAllChannelsRunning { testAllChannelsLock.Unlock() @@ -988,7 +988,11 @@ func testChannels(channels []*model.Channel, testUserID int, notify bool, allowD var runtimeState channelhealth.RuntimeState var hasRuntimeState bool if setting.TimeoutAutoDisableEnabled { - runtimeState, hasRuntimeState = loadChannelHealthRuntimeState(channel.Id) + if runtimeStates != nil { + runtimeState, hasRuntimeState = runtimeStates[channel.Id] + } else { + runtimeState, hasRuntimeState = loadChannelHealthRuntimeState(channel.Id) + } } runtimeRecoveryProbe := hasRuntimeState && runtimeState.RuntimeDisabled && (runtimeState.DisabledUntil.IsZero() || !time.Now().Before(runtimeState.DisabledUntil)) && @@ -1087,7 +1091,7 @@ func testAllChannels(notify bool) error { if getChannelErr != nil { return getChannelErr } - return testChannels(selectChannelsForAutomaticTest(channels, operation_setting.ChannelTestModeScheduledAll), testUserID, notify, true) + return testChannels(selectChannelsForAutomaticTest(channels, operation_setting.ChannelTestModeScheduledAll), testUserID, notify, true, nil) } func testAutoDisabledChannels(notify bool) error { @@ -1099,7 +1103,8 @@ func testAutoDisabledChannels(notify bool) error { if getChannelErr != nil { return getChannelErr } - return testChannels(selectChannelsForRuntimeRecovery(channels), testUserID, notify, false) + selected, runtimeStates := selectChannelsForRuntimeRecoveryWithStates(channels) + return testChannels(selected, testUserID, notify, false, runtimeStates) } // Runtime recovery remains scheduled when the legacy all-channel test is @@ -1109,7 +1114,7 @@ func testRuntimeDisabledChannels() error { if err != nil { return err } - selected := selectChannelsForRuntimeRecovery(channels) + selected, runtimeStates := selectChannelsForRuntimeRecoveryWithStates(channels) runtimeOnly := make([]*model.Channel, 0, len(selected)) for _, channel := range selected { if channel.Status == common.ChannelStatusEnabled { @@ -1123,7 +1128,7 @@ func testRuntimeDisabledChannels() error { if err != nil { return err } - return testChannels(runtimeOnly, testUserID, false, false) + return testChannels(runtimeOnly, testUserID, false, false, runtimeStates) } func loadChannelHealthRuntimeState(channelID int) (channelhealth.RuntimeState, bool) { @@ -1137,9 +1142,38 @@ func loadChannelHealthRuntimeState(channelID int) (channelhealth.RuntimeState, b } func selectChannelsForRuntimeRecovery(channels []*model.Channel) []*model.Channel { + selected, _ := selectChannelsForRuntimeRecoveryWithStates(channels) + return selected +} + +func selectChannelsForRuntimeRecoveryWithStates(channels []*model.Channel) ([]*model.Channel, map[int]channelhealth.RuntimeState) { selected := make([]*model.Channel, 0, len(channels)) now := time.Now() setting := operation_setting.GetMonitorSetting() + if setting == nil { + return selected, nil + } + shouldLoadRuntimeStates := setting.TimeoutAutoDisableEnabled && + setting.TimeoutAutoDisableRecoveryMode == operation_setting.TimeoutAutoDisableRecoveryProbe && + setting.TimeoutAutoDisableDurationSeconds > 0 + runtimeStates := make(map[int]channelhealth.RuntimeState) + if shouldLoadRuntimeStates { + candidateIDs := make([]int, 0, len(channels)) + for _, channel := range channels { + if channel != nil && channel.Status != common.ChannelStatusManuallyDisabled && + channel.Status != common.ChannelStatusAutoDisabled { + candidateIDs = append(candidateIDs, channel.Id) + } + } + if len(candidateIDs) > 0 { + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + loaded, err := channelhealth.LoadRuntimeStates(ctx, candidateIDs) + cancel() + if err == nil { + runtimeStates = loaded + } + } + } for _, channel := range channels { if channel == nil || channel.Status == common.ChannelStatusManuallyDisabled { continue @@ -1148,17 +1182,16 @@ func selectChannelsForRuntimeRecovery(channels []*model.Channel) []*model.Channe selected = append(selected, channel) continue } - if !setting.TimeoutAutoDisableEnabled || - setting.TimeoutAutoDisableRecoveryMode != operation_setting.TimeoutAutoDisableRecoveryProbe || - setting.TimeoutAutoDisableDurationSeconds <= 0 { + if !shouldLoadRuntimeStates { continue } - state, ok := loadChannelHealthRuntimeState(channel.Id) - if ok && (state.DisabledUntil.IsZero() || !now.Before(state.DisabledUntil)) { + state, ok := runtimeStates[channel.Id] + if ok && state.RuntimeDisabled && + (state.DisabledUntil.IsZero() || !now.Before(state.DisabledUntil)) { selected = append(selected, channel) } } - return selected + return selected, runtimeStates } // RecoverChannelHealthRuntime clears only the timeout strategy's Redis state. diff --git a/controller/channel_health_recovery_test.go b/controller/channel_health_recovery_test.go index 27ccc30b..0f8ffa0e 100644 --- a/controller/channel_health_recovery_test.go +++ b/controller/channel_health_recovery_test.go @@ -42,11 +42,15 @@ func TestSelectChannelsForRuntimeRecoveryOnlyProbesDueEligibleChannels(t *testin }).Err() require.NoError(t, err) } + require.NoError(t, common.RDB.HSet(context.Background(), channelhealth.RuntimeStateKey(5), map[string]interface{}{ + "runtime_disabled": "false", + }).Err()) channels := []*model.Channel{ {Id: 1, Status: common.ChannelStatusEnabled}, {Id: 2, Status: common.ChannelStatusEnabled}, {Id: 3, Status: common.ChannelStatusManuallyDisabled}, {Id: 4, Status: common.ChannelStatusAutoDisabled}, + {Id: 5, Status: common.ChannelStatusEnabled}, } selected := selectChannelsForRuntimeRecovery(channels) require.Len(t, selected, 2) diff --git a/model/channel_health_routing.go b/model/channel_health_routing.go index 167e7a83..8cda0360 100644 --- a/model/channel_health_routing.go +++ b/model/channel_health_routing.go @@ -38,6 +38,11 @@ func channelHealthRoutingEnabled() bool { return setting != nil && (setting.AutoPriorityDemotionEnabled || setting.TimeoutAutoDisableEnabled) } +func runtimeDisableRoutingEnabled() bool { + setting := operation_setting.GetMonitorSetting() + return setting != nil && setting.TimeoutAutoDisableEnabled +} + func loadChannelRuntimeStates(channelIDs []int) map[int]channelhealth.RuntimeState { if !channelHealthRoutingEnabled() || len(channelIDs) == 0 { return nil @@ -109,6 +114,7 @@ func buildChannelRoutingCandidates(channels []*Channel) []channelRoutingCandidat } } states := loadChannelRuntimeStates(channelIDs) + excludeRuntimeDisabled := runtimeDisableRoutingEnabled() now := time.Now() candidates := make([]channelRoutingCandidate, 0, len(channels)) for _, channel := range channels { @@ -117,7 +123,7 @@ func buildChannelRoutingCandidates(channels []*Channel) []channelRoutingCandidat } priority := channel.GetPriority() if state, ok := states[channel.Id]; ok { - if state.RuntimeDisabled { + if state.RuntimeDisabled && excludeRuntimeDisabled { continue } priority = channelhealth.EffectivePriority(priority, state, now) @@ -133,12 +139,13 @@ func buildAbilityRoutingCandidates(abilities []Ability) []abilityRoutingCandidat channelIDs = append(channelIDs, ability.ChannelId) } states := loadChannelRuntimeStates(channelIDs) + excludeRuntimeDisabled := runtimeDisableRoutingEnabled() now := time.Now() candidates := make([]abilityRoutingCandidate, 0, len(abilities)) for _, ability := range abilities { priority := abilityPriority(ability) if state, ok := states[ability.ChannelId]; ok { - if state.RuntimeDisabled { + if state.RuntimeDisabled && excludeRuntimeDisabled { continue } priority = channelhealth.EffectivePriority(priority, state, now) diff --git a/model/channel_health_routing_test.go b/model/channel_health_routing_test.go index be874e0a..3190d835 100644 --- a/model/channel_health_routing_test.go +++ b/model/channel_health_routing_test.go @@ -96,6 +96,27 @@ func TestAbilityRoutingExcludesRuntimeDisabledChannel(t *testing.T) { require.Equal(t, 2, selected) } +func TestAbilityRoutingIgnoresRuntimeDisablementWhenAutoDisableIsOff(t *testing.T) { + setting := operation_setting.MonitorSetting{ + AutoPriorityDemotionEnabled: true, + TimeoutAutoDisableEnabled: false, + StreamingFirstResultTimeoutSeconds: 1, + } + withChannelRoutingTestState(t, setting) + require.NoError(t, maxcommon.RDB.HSet(context.Background(), channelhealth.RuntimeStateKey(1), map[string]interface{}{ + "runtime_disabled": "true", + }).Err()) + + baseHigh, baseLow := int64(100), int64(90) + selected, ok, err := selectChannelIdFromAbilities([]Ability{ + {ChannelId: 1, Priority: &baseHigh, Weight: 100}, + {ChannelId: 2, Priority: &baseLow, Weight: 100}, + }, 0, nil) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, 1, selected) +} + func TestAbilityRoutingFailsOpenWhenRedisUnavailable(t *testing.T) { oldSetting := *operation_setting.GetMonitorSetting() oldRDB, oldEnabled := maxcommon.RDB, maxcommon.RedisEnabled diff --git a/pkg/channelhealth/runtime.go b/pkg/channelhealth/runtime.go index 144a36af..4c198f0e 100644 --- a/pkg/channelhealth/runtime.go +++ b/pkg/channelhealth/runtime.go @@ -24,6 +24,7 @@ const ( TimeoutKindFirstResponse = "first_response" runtimeStateKeyPrefix = "maxapi:channel-timeout:v1:state:" + runtimeStateIndexKey = "maxapi:channel-timeout:v1:index" runtimeDedupeKeyPrefix = "maxapi:channel-timeout:v1:dedupe:" runtimeStateTTL = 7 * 24 * time.Hour runtimeDecisionTimeout = 100 * time.Millisecond @@ -100,6 +101,10 @@ func isSuccessfulSample(evidence AttemptEvidence) bool { return evidence.Success && evidence.TimeoutKind == TimeoutKindFirstResponse } +func isRuntimeRecoveryEvent(event string) bool { + return event == "manual_recovery" || strings.HasPrefix(event, "recovery_probe_") +} + func advancedSampleGateEnabled(setting *operation_setting.MonitorSetting) bool { return setting != nil && (setting.TimeoutAutoDisableMinimumSamples > 0 || setting.TimeoutAutoDisableRatioPercent > 0) } @@ -263,6 +268,18 @@ func ListRuntimeStates(ctx context.Context) ([]RuntimeState, error) { if ctx == nil { ctx = context.Background() } + members, err := maxcommon.RDB.SMembers(ctx, runtimeStateIndexKey).Result() + if err != nil { + return nil, err + } + if len(members) > 0 { + return listRuntimeStatesByMembers(ctx, members) + } + + // States written before the index was introduced are discovered once by a + // bounded legacy scan and added to the index below. New state transitions add + // their channel ID immediately after committing, so normal requests never + // scan the shared keyspace. keys := make([]string, 0, 32) var cursor uint64 for { @@ -279,24 +296,50 @@ func ListRuntimeStates(ctx context.Context) ([]RuntimeState, error) { if len(keys) == 0 { return []RuntimeState{}, nil } - pipe := maxcommon.RDB.Pipeline() - commands := make([]*redis.StringStringMapCmd, 0, len(keys)) + members = make([]string, 0, len(keys)) for _, key := range keys { - commands = append(commands, pipe.HGetAll(ctx, key)) + members = append(members, strings.TrimPrefix(key, runtimeStateKeyPrefix)) } - if _, err := pipe.Exec(ctx); err != nil { + states, err := listRuntimeStatesByMembers(ctx, members) + if err != nil { return nil, err } - states := make([]RuntimeState, 0, len(commands)) - for index, command := range commands { - channelID, err := strconv.Atoi(strings.TrimPrefix(keys[index], runtimeStateKeyPrefix)) + validMembers := make([]string, 0, len(states)) + for _, state := range states { + validMembers = append(validMembers, strconv.Itoa(state.ChannelID)) + } + if len(validMembers) > 0 { + _ = maxcommon.RDB.SAdd(ctx, runtimeStateIndexKey, validMembers).Err() + } + return states, nil +} + +func listRuntimeStatesByMembers(ctx context.Context, members []string) ([]RuntimeState, error) { + pipe := maxcommon.RDB.Pipeline() + commands := make([]*redis.StringStringMapCmd, 0, len(members)) + validMembers := make([]string, 0, len(members)) + for _, member := range members { + channelID, err := strconv.Atoi(strings.TrimSpace(member)) if err != nil || channelID <= 0 { continue } + validMembers = append(validMembers, strconv.Itoa(channelID)) + commands = append(commands, pipe.HGetAll(ctx, runtimeStateKey(channelID))) + } + if _, err := pipe.Exec(ctx); err != nil { + return nil, err + } + states := make([]RuntimeState, 0, len(commands)) + for index, command := range commands { + channelID, _ := strconv.Atoi(validMembers[index]) fields, err := command.Result() if err != nil && !errors.Is(err, redis.Nil) { return nil, err } + if len(fields) == 0 { + _ = maxcommon.RDB.SRem(ctx, runtimeStateIndexKey, validMembers[index]).Err() + continue + } states = append(states, decodeRuntimeState(channelID, fields)) } return states, nil @@ -348,6 +391,11 @@ func EvaluateTimeoutGuard(ctx context.Context, evidence AttemptEvidence, setting } state := decodeRuntimeState(evidence.ChannelID, fields) result.State = state + if isRuntimeRecoveryEvent(state.LastEvent) && !state.LastObservedAt.IsZero() && now.Before(state.LastObservedAt) { + // A timeout observed before the last recovery must not be allowed to + // reapply a penalty after the recovery transaction wins the watch. + return nil + } if deduped > 0 { result.Deduplicated = true return nil @@ -386,7 +434,8 @@ func EvaluateTimeoutGuard(ctx context.Context, evidence AttemptEvidence, setting state.Penalty = int64(setting.PriorityDeduction) if setting.RecoveryMode == operation_setting.RecoveryModeAutomatic { until := now.Add(time.Duration(setting.PenaltyCooldownSeconds) * time.Second) - if setting.ExtendOnRepeatTimeout && previouslyActive && until.Before(state.PenaltyUntil) { + if previouslyActive && !state.PenaltyUntil.IsZero() && + (!setting.ExtendOnRepeatTimeout || until.Before(state.PenaltyUntil)) { until = state.PenaltyUntil } state.PenaltyUntil = until @@ -444,6 +493,7 @@ func EvaluateTimeoutGuard(ctx context.Context, evidence AttemptEvidence, setting } return EvaluationResult{State: emptyRuntimeState(evidence.ChannelID)}, err } + _ = maxcommon.RDB.SAdd(ctx, runtimeStateIndexKey, strconv.Itoa(evidence.ChannelID)).Err() return result, nil } @@ -558,6 +608,7 @@ func RecoverRuntimeState(ctx context.Context, channelID int, probeID string, suc } return EvaluationResult{State: emptyRuntimeState(channelID)}, err } + _ = maxcommon.RDB.SAdd(decisionCtx, runtimeStateIndexKey, strconv.Itoa(channelID)).Err() return result, nil } @@ -607,6 +658,7 @@ func RestoreRuntimeState(ctx context.Context, channelID int) (EvaluationResult, } return EvaluationResult{State: emptyRuntimeState(channelID)}, err } + _ = maxcommon.RDB.SAdd(decisionCtx, runtimeStateIndexKey, strconv.Itoa(channelID)).Err() return result, nil } @@ -621,7 +673,7 @@ func EffectivePriority(base int64, state RuntimeState, now time.Time) int64 { return base } effective := base - state.Penalty - if effective < 0 { + if effective < 0 && base >= 0 { return 0 } return effective diff --git a/pkg/channelhealth/runtime_test.go b/pkg/channelhealth/runtime_test.go index 78044118..8a45c180 100644 --- a/pkg/channelhealth/runtime_test.go +++ b/pkg/channelhealth/runtime_test.go @@ -286,6 +286,77 @@ func TestEffectivePriorityExpiresAndClampsAtZero(t *testing.T) { require.Equal(t, int64(80), EffectivePriority(100, state, time.Now())) require.Equal(t, int64(100), EffectivePriority(100, state, time.Now().Add(2*time.Minute))) require.Equal(t, int64(0), EffectivePriority(10, state, time.Now())) + require.Equal(t, int64(-25), EffectivePriority(-5, state, time.Now())) +} + +func TestEvaluateTimeoutGuardPreservesPenaltyDeadlineWhenRepeatExtensionDisabled(t *testing.T) { + withRuntimeRedis(t) + setting := testRuntimeSetting() + setting.ExtendOnRepeatTimeout = false + setting.TimeoutAutoDisableCount = 100 + now := time.Now() + + first, err := EvaluateTimeoutGuard(context.Background(), AttemptEvidence{ + ChannelID: 936, + AttemptID: "non-extending-first", + RequestMode: RequestModeStreaming, + TimeoutKind: TimeoutKindStreamingFirstResult, + TimeoutEligible: true, + AutoBan: true, + ObservedAt: now, + }, setting) + require.NoError(t, err) + + second, err := EvaluateTimeoutGuard(context.Background(), AttemptEvidence{ + ChannelID: 936, + AttemptID: "non-extending-repeat", + RequestMode: RequestModeStreaming, + TimeoutKind: TimeoutKindStreamingFirstResult, + TimeoutEligible: true, + AutoBan: true, + ObservedAt: now.Add(10 * time.Second), + }, setting) + require.NoError(t, err) + require.WithinDuration(t, first.State.PenaltyUntil, second.State.PenaltyUntil, time.Millisecond) +} + +func TestEvaluateTimeoutGuardIgnoresTimeoutObservedBeforeRecovery(t *testing.T) { + withRuntimeRedis(t) + setting := testRuntimeSetting() + staleObservedAt := time.Now().Add(-time.Second) + + first, err := EvaluateTimeoutGuard(context.Background(), AttemptEvidence{ + ChannelID: 936, + AttemptID: "stale-before-recovery", + RequestMode: RequestModeStreaming, + TimeoutKind: TimeoutKindStreamingFirstResult, + TimeoutEligible: true, + AutoBan: true, + ObservedAt: staleObservedAt, + }, setting) + require.NoError(t, err) + require.True(t, first.Applied) + + recovered, err := RestoreRuntimeState(context.Background(), 936) + require.NoError(t, err) + require.True(t, recovered.Applied) + require.Equal(t, "manual_recovery", recovered.State.LastEvent) + + stale, err := EvaluateTimeoutGuard(context.Background(), AttemptEvidence{ + ChannelID: 936, + AttemptID: "stale-after-recovery", + RequestMode: RequestModeStreaming, + TimeoutKind: TimeoutKindStreamingFirstResult, + TimeoutEligible: true, + AutoBan: true, + ObservedAt: staleObservedAt, + }, setting) + require.NoError(t, err) + require.False(t, stale.Applied) + require.False(t, stale.Deduplicated) + require.Equal(t, "manual_recovery", stale.State.LastEvent) + require.Zero(t, stale.State.Penalty) + require.False(t, stale.State.RuntimeDisabled) } func TestStreamIdleTimeoutDoesNotCountOrDemote(t *testing.T) { @@ -326,6 +397,9 @@ func TestListRuntimeStatesRebuildsTheSmartOpsProjection(t *testing.T) { require.Equal(t, 936, states[0].ChannelID) require.Equal(t, int64(10), states[0].Penalty) require.Equal(t, int64(1), states[0].TimeoutCount[RequestModeStreaming]) + indexed, err := maxcommon.RDB.SIsMember(context.Background(), runtimeStateIndexKey, "936").Result() + require.NoError(t, err) + require.True(t, indexed) } func TestRecoverRuntimeStateRequiresConfiguredSuccessfulProbes(t *testing.T) { diff --git a/web/default/src/features/smart-ops/channel-health/index.tsx b/web/default/src/features/smart-ops/channel-health/index.tsx index ba64b465..53f5b2bb 100644 --- a/web/default/src/features/smart-ops/channel-health/index.tsx +++ b/web/default/src/features/smart-ops/channel-health/index.tsx @@ -962,7 +962,7 @@ export function ChannelHealth(): ReactElement { size='sm' disabled={recoveryMutation.isPending} onClick={() => - void recoveryMutation.mutateAsync( + recoveryMutation.mutate( channelIdFromAlert(alert) ) } From d822eb60c44b7aecfea2cf457afc5ded152268a3 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Sat, 3 Oct 2026 01:47:01 +0800 Subject: [PATCH 04/12] fix: close channel health follow-up review findings --- pkg/channelhealth/runtime.go | 56 +++++-- pkg/channelhealth/runtime_test.go | 30 ++++ .../common/channel_health_observation_test.go | 40 +++++ relay/common/relay_info.go | 19 ++- service/smart_ops_alert.go | 87 ++++++++-- service/smart_ops_alert_delivery_test.go | 20 +++ setting/config/config.go | 3 + setting/operation_setting/monitor_setting.go | 59 ++++--- .../operation_setting/monitor_setting_test.go | 19 +++ web/default/src/i18n/locales/fr.json | 156 +++++++++--------- web/default/src/i18n/locales/ja.json | 156 +++++++++--------- web/default/src/i18n/locales/ru.json | 156 +++++++++--------- web/default/src/i18n/locales/vi.json | 156 +++++++++--------- 13 files changed, 591 insertions(+), 366 deletions(-) diff --git a/pkg/channelhealth/runtime.go b/pkg/channelhealth/runtime.go index 4c198f0e..4e08252e 100644 --- a/pkg/channelhealth/runtime.go +++ b/pkg/channelhealth/runtime.go @@ -23,12 +23,13 @@ const ( TimeoutKindStreamIdle = "stream_idle_timeout" TimeoutKindFirstResponse = "first_response" - runtimeStateKeyPrefix = "maxapi:channel-timeout:v1:state:" - runtimeStateIndexKey = "maxapi:channel-timeout:v1:index" - runtimeDedupeKeyPrefix = "maxapi:channel-timeout:v1:dedupe:" - runtimeStateTTL = 7 * 24 * time.Hour - runtimeDecisionTimeout = 100 * time.Millisecond - runtimeWatchAttempts = 32 + runtimeStateKeyPrefix = "maxapi:channel-timeout:v1:state:" + runtimeStateIndexKey = "maxapi:channel-timeout:v1:index" + runtimeStateIndexMigratedKey = "maxapi:channel-timeout:v1:index:migrated" + runtimeDedupeKeyPrefix = "maxapi:channel-timeout:v1:dedupe:" + runtimeStateTTL = 7 * 24 * time.Hour + runtimeDecisionTimeout = 100 * time.Millisecond + runtimeWatchAttempts = 32 ) var ErrRuntimeUnavailable = errors.New("channel timeout runtime is unavailable") @@ -275,13 +276,22 @@ func ListRuntimeStates(ctx context.Context) ([]RuntimeState, error) { if len(members) > 0 { return listRuntimeStatesByMembers(ctx, members) } + migrated, err := maxcommon.RDB.Exists(ctx, runtimeStateIndexMigratedKey).Result() + if err != nil { + return nil, err + } + if migrated > 0 { + return []RuntimeState{}, nil + } - // States written before the index was introduced are discovered once by a - // bounded legacy scan and added to the index below. New state transitions add - // their channel ID immediately after committing, so normal requests never - // scan the shared keyspace. + // States written before the index was introduced are discovered by a bounded + // legacy scan and added to the index below. A complete scan records a marker; + // an interrupted scan remains eligible for a later migration attempt. New + // state transitions add their channel ID in the same Redis transaction, so + // normal requests never scan the shared keyspace after migration completes. keys := make([]string, 0, 32) var cursor uint64 + scanComplete := false for { batch, next, err := maxcommon.RDB.Scan(ctx, cursor, runtimeStateKeyPrefix+"*", 200).Result() if err != nil { @@ -289,11 +299,20 @@ func ListRuntimeStates(ctx context.Context) ([]RuntimeState, error) { } keys = append(keys, batch...) cursor = next - if cursor == 0 || len(keys) >= 1000 { + if cursor == 0 { + scanComplete = true + break + } + if len(keys) >= 1000 { break } } if len(keys) == 0 { + if scanComplete { + if err := maxcommon.RDB.Set(ctx, runtimeStateIndexMigratedKey, "1", 0).Err(); err != nil { + return nil, err + } + } return []RuntimeState{}, nil } members = make([]string, 0, len(keys)) @@ -309,7 +328,14 @@ func ListRuntimeStates(ctx context.Context) ([]RuntimeState, error) { validMembers = append(validMembers, strconv.Itoa(state.ChannelID)) } if len(validMembers) > 0 { - _ = maxcommon.RDB.SAdd(ctx, runtimeStateIndexKey, validMembers).Err() + if err := maxcommon.RDB.SAdd(ctx, runtimeStateIndexKey, validMembers).Err(); err != nil { + return nil, err + } + } + if scanComplete { + if err := maxcommon.RDB.Set(ctx, runtimeStateIndexMigratedKey, "1", 0).Err(); err != nil { + return nil, err + } } return states, nil } @@ -475,6 +501,7 @@ func EvaluateTimeoutGuard(ctx context.Context, evidence AttemptEvidence, setting result.State = state pipe := tx.TxPipeline() pipe.HSet(ctx, key, encodeRuntimeState(state)) + pipe.SAdd(ctx, runtimeStateIndexKey, strconv.Itoa(evidence.ChannelID)) pipe.Set(ctx, dedupeKey, "1", windowTTL) if state.RuntimeDisabled && (setting.TimeoutAutoDisableDurationSeconds == 0 || @@ -493,7 +520,6 @@ func EvaluateTimeoutGuard(ctx context.Context, evidence AttemptEvidence, setting } return EvaluationResult{State: emptyRuntimeState(evidence.ChannelID)}, err } - _ = maxcommon.RDB.SAdd(ctx, runtimeStateIndexKey, strconv.Itoa(evidence.ChannelID)).Err() return result, nil } @@ -597,6 +623,7 @@ func RecoverRuntimeState(ctx context.Context, channelID int, probeID string, suc result.State = state pipe := tx.TxPipeline() pipe.HSet(decisionCtx, key, encodeRuntimeState(state)) + pipe.SAdd(decisionCtx, runtimeStateIndexKey, strconv.Itoa(channelID)) pipe.Set(decisionCtx, dedupeKey, "1", probeTTL) pipe.Expire(decisionCtx, key, runtimeStateTTL) _, err = pipe.Exec(decisionCtx) @@ -608,7 +635,6 @@ func RecoverRuntimeState(ctx context.Context, channelID int, probeID string, suc } return EvaluationResult{State: emptyRuntimeState(channelID)}, err } - _ = maxcommon.RDB.SAdd(decisionCtx, runtimeStateIndexKey, strconv.Itoa(channelID)).Err() return result, nil } @@ -648,6 +674,7 @@ func RestoreRuntimeState(ctx context.Context, channelID int) (EvaluationResult, result.Applied = true pipe := tx.TxPipeline() pipe.HSet(decisionCtx, key, encodeRuntimeState(state)) + pipe.SAdd(decisionCtx, runtimeStateIndexKey, strconv.Itoa(channelID)) pipe.Expire(decisionCtx, key, runtimeStateTTL) _, err = pipe.Exec(decisionCtx) return err @@ -658,7 +685,6 @@ func RestoreRuntimeState(ctx context.Context, channelID int) (EvaluationResult, } return EvaluationResult{State: emptyRuntimeState(channelID)}, err } - _ = maxcommon.RDB.SAdd(decisionCtx, runtimeStateIndexKey, strconv.Itoa(channelID)).Err() return result, nil } diff --git a/pkg/channelhealth/runtime_test.go b/pkg/channelhealth/runtime_test.go index 8a45c180..4a088b8f 100644 --- a/pkg/channelhealth/runtime_test.go +++ b/pkg/channelhealth/runtime_test.go @@ -402,6 +402,36 @@ func TestListRuntimeStatesRebuildsTheSmartOpsProjection(t *testing.T) { require.True(t, indexed) } +func TestListRuntimeStatesMarksLegacyScanComplete(t *testing.T) { + withRuntimeRedis(t) + key := RuntimeStateKey(937) + _, err := maxcommon.RDB.HSet(context.Background(), key, map[string]interface{}{ + "channel_id": "937", + "penalty": "5", + }).Result() + require.NoError(t, err) + + states, err := ListRuntimeStates(context.Background()) + require.NoError(t, err) + require.Len(t, states, 1) + require.Equal(t, 937, states[0].ChannelID) + migrated, err := maxcommon.RDB.Exists(context.Background(), runtimeStateIndexMigratedKey).Result() + require.NoError(t, err) + require.Equal(t, int64(1), migrated) + require.NoError(t, maxcommon.RDB.SRem(context.Background(), runtimeStateIndexKey, "937").Err()) + + // A stray legacy-shaped key does not restart a full keyspace scan after the + // migration marker has been recorded. + _, err = maxcommon.RDB.HSet(context.Background(), RuntimeStateKey(938), map[string]interface{}{ + "channel_id": "938", + "penalty": "7", + }).Result() + require.NoError(t, err) + states, err = ListRuntimeStates(context.Background()) + require.NoError(t, err) + require.Empty(t, states) +} + func TestRecoverRuntimeStateRequiresConfiguredSuccessfulProbes(t *testing.T) { withRuntimeRedis(t) setting := testRuntimeSetting() diff --git a/relay/common/channel_health_observation_test.go b/relay/common/channel_health_observation_test.go index 22960b28..2d0395e8 100644 --- a/relay/common/channel_health_observation_test.go +++ b/relay/common/channel_health_observation_test.go @@ -1,10 +1,15 @@ package common import ( + "context" "testing" "time" maxcommon "github.com/MAX-API-Next/MAX-API/common" + "github.com/MAX-API-Next/MAX-API/pkg/channelhealth" + "github.com/MAX-API-Next/MAX-API/setting/operation_setting" + "github.com/alicebob/miniredis/v2" + "github.com/go-redis/redis/v8" "github.com/stretchr/testify/require" ) @@ -55,3 +60,38 @@ func TestRelayInfoFirstResponseEvaluationHasBoundedGrace(t *testing.T) { time.Sleep(firstResponseEvaluationGrace + 10*time.Millisecond) require.True(t, pending.FirstResponseDeadlineExpired()) } + +func TestRelayInfoTransportTimeoutHonorsConfiguredThreshold(t *testing.T) { + server := miniredis.RunT(t) + oldRDB, oldEnabled := maxcommon.RDB, maxcommon.RedisEnabled + maxcommon.RDB = redis.NewClient(&redis.Options{Addr: server.Addr()}) + maxcommon.RedisEnabled = true + t.Cleanup(func() { + _ = maxcommon.RDB.Close() + maxcommon.RDB, maxcommon.RedisEnabled = oldRDB, oldEnabled + }) + + oldSetting := *operation_setting.GetMonitorSetting() + *operation_setting.GetMonitorSetting() = operation_setting.MonitorSetting{ + AutoPriorityDemotionEnabled: true, + StreamingFirstResultTimeoutSeconds: 2, + PriorityDeduction: 10, + PenaltyCooldownSeconds: 60, + } + t.Cleanup(func() { *operation_setting.GetMonitorSetting() = oldSetting }) + + start := time.Now().Add(-100 * time.Millisecond) + info := &RelayInfo{ + StartTime: start, + IsStream: true, + ChannelMeta: &ChannelMeta{ChannelId: 936, ChannelAutoBan: true}, + channelAttemptStartTime: start, + } + info.RecordChannelTimeout("transport_timeout") + time.Sleep(50 * time.Millisecond) + + state, err := channelhealth.LoadRuntimeState(context.Background(), 936) + require.NoError(t, err) + require.Zero(t, state.Penalty) + require.Zero(t, state.TimeoutCount[channelhealth.RequestModeStreaming]) +} diff --git a/relay/common/relay_info.go b/relay/common/relay_info.go index 1a0b6f83..a70e4e8b 100644 --- a/relay/common/relay_info.go +++ b/relay/common/relay_info.go @@ -922,7 +922,24 @@ func (info *RelayInfo) RecordChannelTimeout(event string) { return } timeoutKind := event + observedAt := time.Now() if event == "transport_timeout" || event == "request_timeout" { + start := info.channelAttemptStartTime + if start.IsZero() { + start = info.StartTime + } + setting := operation_setting.GetMonitorSetting() + threshold := 0 + if setting != nil { + if info.IsStream { + threshold = setting.StreamingFirstResultTimeoutSeconds + } else { + threshold = setting.NonStreamingResponseTimeoutSeconds + } + } + if start.IsZero() || threshold <= 0 || observedAt.Sub(start) < time.Duration(threshold)*time.Second { + return + } if info.IsStream && !info.channelFirstResponseRecorded.Load() { timeoutKind = channelhealth.TimeoutKindStreamingFirstResult } else if !info.IsStream { @@ -948,7 +965,7 @@ func (info *RelayInfo) RecordChannelTimeout(event string) { TimeoutKind: timeoutKind, TimeoutEligible: true, AutoBan: info.ChannelAutoBan, - ObservedAt: time.Now(), + ObservedAt: observedAt, } settingSnapshot := *setting go func(evidence channelhealth.AttemptEvidence, setting operation_setting.MonitorSetting) { diff --git a/service/smart_ops_alert.go b/service/smart_ops_alert.go index 4f950d4d..bda0770e 100644 --- a/service/smart_ops_alert.go +++ b/service/smart_ops_alert.go @@ -1216,6 +1216,7 @@ type smartOpsAlertDeliveryProjection struct { status string attempts int detail string + eventAt time.Time } var smartOpsAlertDeliveryProjectionQueue struct { @@ -1228,7 +1229,7 @@ func startSmartOpsAlertDeliveryProjectionWorker() { smartOpsAlertDeliveryProjectionQueue.queue = make(chan smartOpsAlertDeliveryProjection, smartOpsAlertQueueSize) go func() { for projection := range smartOpsAlertDeliveryProjectionQueue.queue { - recordSmartOpsAlertDeliveryState(projection.alert, projection.status, projection.attempts, projection.detail) + recordSmartOpsAlertDeliveryStateAt(projection.alert, projection.status, projection.attempts, projection.detail, projection.eventAt) } }() }) @@ -1330,10 +1331,15 @@ func enqueueSmartOpsAlertNotification(alert SmartOpsAlert) { }) if !smartOpsAlertNotificationQueue.pool.enqueue(alert) { + eventAt := alert.ObservedAt + if eventAt.IsZero() { + eventAt = time.Now() + } if !enqueueSmartOpsAlertDeliveryProjection(smartOpsAlertDeliveryProjection{ - alert: alert, - status: "failed", - detail: "notification_queue_full", + alert: alert, + status: "failed", + detail: "notification_queue_full", + eventAt: eventAt, }) { // Both queues are intentionally bounded. Losing this optional // diagnostic projection is preferable to creating an unbounded @@ -1406,10 +1412,36 @@ func clearSmartOpsAlertRepeatKey(alert SmartOpsAlert) { _, _ = common.RDB.Eval(ctx, smartOpsAlertRepeatReleaseScript, []string{smartOpsAlertRepeatKey(alert)}, alert.repeatLockToken).Result() } +func smartOpsAlertDeliveryStatusRank(status string) int { + switch status { + case "queued": + return 10 + case "sending": + return 20 + case "sent", "skipped_unconfigured": + return 40 + case "skipped_repeat": + return 50 + default: + return 30 + } +} + // recordSmartOpsAlertDeliveryState stores a bounded delivery projection in // Redis. It is diagnostic and recoverable; failures must never change the -// alert transition or block the notification worker. +// alert transition or block the notification worker. Status updates use the +// source observation time and a phase rank so an older queued/failed event +// cannot overwrite a newer terminal state when projection queues finish out of +// order. func recordSmartOpsAlertDeliveryState(alert SmartOpsAlert, status string, attempts int, detail string) { + eventAt := alert.ObservedAt + if eventAt.IsZero() { + eventAt = time.Now() + } + recordSmartOpsAlertDeliveryStateAt(alert, status, attempts, detail, eventAt) +} + +func recordSmartOpsAlertDeliveryStateAt(alert SmartOpsAlert, status string, attempts int, detail string, eventAt time.Time) { if !common.RedisEnabled || common.RDB == nil || status == "" { return } @@ -1419,21 +1451,42 @@ func recordSmartOpsAlertDeliveryState(alert SmartOpsAlert, status string, attemp if len(detail) > 512 { detail = detail[:512] } + if eventAt.IsZero() { + eventAt = time.Now() + } ctx, cancel := context.WithTimeout(context.Background(), smartOpsAlertDeliveryTimeout) defer cancel() + key := smartOpsAlertDeliveryKey(alert) fields := map[string]interface{}{ - "alert_key": alert.Key, - "alert_status": alert.Status, - "alert_node": alert.Node, - "delivery_status": status, - "attempts": strconv.Itoa(attempts), - "error": detail, - "updated_at": time.Now().UTC().Format(time.RFC3339Nano), - } - pipe := common.RDB.TxPipeline() - pipe.HSet(ctx, smartOpsAlertDeliveryKey(alert), fields) - pipe.Expire(ctx, smartOpsAlertDeliveryKey(alert), smartOpsAlertDeliveryTTL) - _, _ = pipe.Exec(ctx) + "alert_key": alert.Key, + "alert_status": alert.Status, + "alert_node": alert.Node, + "delivery_status": status, + "attempts": strconv.Itoa(attempts), + "error": detail, + "delivery_event_at": strconv.FormatInt(eventAt.UnixMilli(), 10), + "delivery_status_rank": strconv.Itoa(smartOpsAlertDeliveryStatusRank(status)), + "updated_at": time.Now().UTC().Format(time.RFC3339Nano), + } + _ = common.RDB.Watch(ctx, func(tx *redis.Tx) error { + current, err := tx.HMGet(ctx, key, "delivery_event_at", "delivery_status_rank").Result() + if err != nil { + return err + } + currentEventAt, _ := strconv.ParseInt(fmt.Sprint(current[0]), 10, 64) + currentRank, _ := strconv.Atoi(fmt.Sprint(current[1])) + incomingEventAt := eventAt.UnixMilli() + incomingRank := smartOpsAlertDeliveryStatusRank(status) + if currentEventAt > incomingEventAt || + (currentEventAt == incomingEventAt && currentRank >= incomingRank) { + return nil + } + pipe := tx.TxPipeline() + pipe.HSet(ctx, key, fields) + pipe.Expire(ctx, key, smartOpsAlertDeliveryTTL) + _, err = pipe.Exec(ctx) + return err + }, key) } func loadSmartOpsAlertDeliveryState(alert *SmartOpsAlert) { diff --git a/service/smart_ops_alert_delivery_test.go b/service/smart_ops_alert_delivery_test.go index ee7f0c17..c4fb27a8 100644 --- a/service/smart_ops_alert_delivery_test.go +++ b/service/smart_ops_alert_delivery_test.go @@ -45,6 +45,26 @@ func TestSmartOpsAlertDeliveryStateIsTraceableInRedis(t *testing.T) { require.False(t, failed.DeliveryUpdatedAt.IsZero()) } +func TestSmartOpsAlertDeliveryProjectionRejectsOlderStatus(t *testing.T) { + withSmartOpsDeliveryRedis(t) + eventAt := time.Now().UTC().Truncate(time.Millisecond) + alert := SmartOpsAlert{ + Key: "channel_timeout_auto_disabled:936", + Node: "node-a", + Status: smartOpsAlertStatusFiring, + ObservedAt: eventAt, + } + + recordSmartOpsAlertDeliveryState(alert, "sent", 1, "") + recordSmartOpsAlertDeliveryState(alert, "queued", 0, "") + recordSmartOpsAlertDeliveryState(alert, "failed", 1, "late queue projection") + + state := alert + loadSmartOpsAlertDeliveryState(&state) + require.Equal(t, "sent", state.DeliveryStatus) + require.Equal(t, 1, state.DeliveryAttempts) +} + func TestDeliverSmartOpsAlertRecordsSentDeliveryState(t *testing.T) { withSmartOpsDeliveryRedis(t) originalDelays := smartOpsAlertRetryDelays diff --git a/setting/config/config.go b/setting/config/config.go index 25f96ae8..bcc8c582 100644 --- a/setting/config/config.go +++ b/setting/config/config.go @@ -286,6 +286,9 @@ func updateConfigFromMap(config interface{}, configMap map[string]string) error field.Set(fresh.Elem()) } } + if normalizer, ok := config.(interface{ Normalize() }); ok { + normalizer.Normalize() + } return nil } diff --git a/setting/operation_setting/monitor_setting.go b/setting/operation_setting/monitor_setting.go index 7b052a50..296e4596 100644 --- a/setting/operation_setting/monitor_setting.go +++ b/setting/operation_setting/monitor_setting.go @@ -82,34 +82,51 @@ func init() { config.GlobalConfig.Register("monitor_setting", &monitorSetting) } +// Normalize applies compatibility defaults after a persisted configuration +// update. Keeping this separate from GetMonitorSetting makes reads side-effect +// free for concurrent relay requests. +func (setting *MonitorSetting) Normalize() { + if setting == nil { + return + } + if setting.ChannelTestMode != ChannelTestModePassiveRecovery { + setting.ChannelTestMode = ChannelTestModeScheduledAll + } + if setting.TimeoutAutoDisableCountScope != TimeoutAutoDisableCountScopeCombined { + setting.TimeoutAutoDisableCountScope = TimeoutAutoDisableCountScopeSameMode + } + if setting.TimeoutAutoDisableRecoveryMode != TimeoutAutoDisableRecoveryManual { + setting.TimeoutAutoDisableRecoveryMode = TimeoutAutoDisableRecoveryProbe + } + if setting.RecoveryMode != RecoveryModeManual { + setting.RecoveryMode = RecoveryModeAutomatic + } +} + func GetMonitorSetting() *MonitorSetting { - if os.Getenv("CHANNEL_TEST_FREQUENCY") != "" { - frequency, err := strconv.Atoi(os.Getenv("CHANNEL_TEST_FREQUENCY")) + frequencyValue, hasFrequency := os.LookupEnv("CHANNEL_TEST_FREQUENCY") + enabledValue, hasEnabled := os.LookupEnv("CHANNEL_TEST_ENABLED") + if !hasFrequency && !hasEnabled { + return &monitorSetting + } + + effective := monitorSetting + if frequencyValue != "" { + frequency, err := strconv.Atoi(frequencyValue) if err == nil && frequency > 0 { - monitorSetting.AutoTestChannelEnabled = true - monitorSetting.AutoTestChannelMinutes = float64(frequency) - monitorSetting.ChannelTestMode = ChannelTestModeScheduledAll + effective.AutoTestChannelEnabled = true + effective.AutoTestChannelMinutes = float64(frequency) + effective.ChannelTestMode = ChannelTestModeScheduledAll } } - if enabled, ok := os.LookupEnv("CHANNEL_TEST_ENABLED"); ok { - parsed, err := strconv.ParseBool(enabled) + if hasEnabled { + parsed, err := strconv.ParseBool(enabledValue) if err == nil { - monitorSetting.AutoTestChannelEnabled = parsed + effective.AutoTestChannelEnabled = parsed } } - if monitorSetting.ChannelTestMode != ChannelTestModePassiveRecovery { - monitorSetting.ChannelTestMode = ChannelTestModeScheduledAll - } - if monitorSetting.TimeoutAutoDisableCountScope != TimeoutAutoDisableCountScopeCombined { - monitorSetting.TimeoutAutoDisableCountScope = TimeoutAutoDisableCountScopeSameMode - } - if monitorSetting.TimeoutAutoDisableRecoveryMode != TimeoutAutoDisableRecoveryManual { - monitorSetting.TimeoutAutoDisableRecoveryMode = TimeoutAutoDisableRecoveryProbe - } - if monitorSetting.RecoveryMode != RecoveryModeManual { - monitorSetting.RecoveryMode = RecoveryModeAutomatic - } - return &monitorSetting + effective.Normalize() + return &effective } // ValidateMonitorSettingOption validates one persisted option without diff --git a/setting/operation_setting/monitor_setting_test.go b/setting/operation_setting/monitor_setting_test.go index ad9bad81..9eb7baff 100644 --- a/setting/operation_setting/monitor_setting_test.go +++ b/setting/operation_setting/monitor_setting_test.go @@ -3,6 +3,7 @@ package operation_setting import ( "testing" + "github.com/MAX-API-Next/MAX-API/setting/config" "github.com/stretchr/testify/require" ) @@ -34,3 +35,21 @@ func TestValidateMonitorSettingOption(t *testing.T) { "monitor_setting.priority_deduction", "-1", )) } + +func TestMonitorSettingNormalizationRunsOnConfigLoad(t *testing.T) { + previous := *GetMonitorSetting() + monitor := config.GlobalConfig.Get("monitor_setting").(*MonitorSetting) + t.Cleanup(func() { *monitor = previous }) + + require.NoError(t, config.GlobalConfig.LoadFromDB(map[string]string{ + "monitor_setting.channel_test_mode": "legacy", + "monitor_setting.timeout_auto_disable_count_scope": "legacy", + "monitor_setting.timeout_auto_disable_recovery_mode": "legacy", + "monitor_setting.recovery_mode": "legacy", + })) + setting := GetMonitorSetting() + require.Equal(t, ChannelTestModeScheduledAll, setting.ChannelTestMode) + require.Equal(t, TimeoutAutoDisableCountScopeSameMode, setting.TimeoutAutoDisableCountScope) + require.Equal(t, TimeoutAutoDisableRecoveryProbe, setting.TimeoutAutoDisableRecoveryMode) + require.Equal(t, RecoveryModeAutomatic, setting.RecoveryMode) +} diff --git a/web/default/src/i18n/locales/fr.json b/web/default/src/i18n/locales/fr.json index b0f57117..b8045e40 100644 --- a/web/default/src/i18n/locales/fr.json +++ b/web/default/src/i18n/locales/fr.json @@ -5747,84 +5747,84 @@ "Enter an exact final quota for each selected task. Each value is checked against its own reserved quota.": "Saisissez un quota final exact pour chaque tâche sélectionnée. Chaque valeur est vérifiée par rapport à son propre quota réservé.", "Apply exact settlements": "Appliquer les règlements exacts", "Allowed range: 0 to {{quota}} ({{quotaPerUnit}} quota = $1).": "Plage autorisée : de 0 à {{quota}} ({{quotaPerUnit}} unités de quota = 1 $).", - "Automatic recovery mode": "Automatic recovery mode", - "Probe then automatic": "Probe then automatic", - "Manual recovery": "Manual recovery", - "Timeout count scope": "Timeout count scope", - "Same request mode": "Same request mode", - "Combined modes": "Combined modes", - "Count streaming and non-streaming timeouts separately or together.": "Count streaming and non-streaming timeouts separately or together.", - "Probe mode recovers after configured successful probes; manual mode requires an administrator.": "Probe mode recovers after configured successful probes; manual mode requires an administrator.", - "Channel recovery failed": "Channel recovery failed", - "Channel recovery requested": "Channel recovery requested", - "Recover channel": "Recover channel", - "We could not load the channel health policy.": "We could not load the channel health policy.", - "Critical": "Critical", - "Channel timeout notifications": "Channel timeout notifications", - "Delivery status": "Delivery status", - "Recovery mode": "Recovery mode", - "Automatic": "Automatic", - "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.", - "Extend penalty on repeated timeout": "Extend penalty on repeated timeout", - "Keep the current penalty deadline when repeated timeouts continue during an active penalty.": "Keep the current penalty deadline when repeated timeouts continue during an active penalty.", - "{{count}} priority point": "{{count}} priority point", - "{{count}} priority points": "{{count}} priority points", - "{{count}} timeout": "{{count}} timeout", - "{{count}} timeouts": "{{count}} timeouts", - "Streaming first-result timeout must be between 0 and 86400 seconds.": "Streaming first-result timeout must be between 0 and 86400 seconds.", - "Non-streaming response timeout must be between 0 and 86400 seconds.": "Non-streaming response timeout must be between 0 and 86400 seconds.", - "Priority deduction must be a non-negative integer.": "Priority deduction must be a non-negative integer.", - "Disable count must be at least 2.": "Disable count must be at least 2.", - "Disable window must be between 60 and 2592000 seconds.": "Disable window must be between 60 and 2592000 seconds.", + "Automatic recovery mode": "Mode de récupération automatique", + "Probe then automatic": "Sonde puis automatique", + "Manual recovery": "Récupération manuelle", + "Timeout count scope": "Portée du comptage des dépassements", + "Same request mode": "Même mode de requête", + "Combined modes": "Modes combinés", + "Count streaming and non-streaming timeouts separately or together.": "Compter séparément ou ensemble les dépassements en streaming et hors streaming.", + "Probe mode recovers after configured successful probes; manual mode requires an administrator.": "Le mode sonde récupère après le nombre de sondes réussies configuré ; le mode manuel nécessite un administrateur.", + "Channel recovery failed": "Échec de la récupération du canal", + "Channel recovery requested": "Récupération du canal demandée", + "Recover channel": "Récupérer le canal", + "We could not load the channel health policy.": "Impossible de charger la politique de santé du canal.", + "Critical": "Critique", + "Channel timeout notifications": "Notifications de dépassement du canal", + "Delivery status": "État de livraison", + "Recovery mode": "Mode de récupération", + "Automatic": "Automatique", + "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "La récupération automatique rétablit la pénalité après son délai de refroidissement ; la récupération manuelle la conserve jusqu’à l’intervention d’un administrateur.", + "Extend penalty on repeated timeout": "Prolonger la pénalité en cas de dépassement répété", + "Keep the current penalty deadline when repeated timeouts continue during an active penalty.": "Conserver l’échéance actuelle de la pénalité lorsque les dépassements répétés se poursuivent pendant une pénalité active.", + "{{count}} priority point": "{{count}} point de priorité", + "{{count}} priority points": "{{count}} points de priorité", + "{{count}} timeout": "{{count}} dépassement", + "{{count}} timeouts": "{{count}} dépassements", + "Streaming first-result timeout must be between 0 and 86400 seconds.": "Le délai du premier résultat en streaming doit être compris entre 0 et 86400 secondes.", + "Non-streaming response timeout must be between 0 and 86400 seconds.": "Le délai de réponse hors streaming doit être compris entre 0 et 86400 secondes.", + "Priority deduction must be a non-negative integer.": "La déduction de priorité doit être un entier non négatif.", + "Disable count must be at least 2.": "Le nombre de désactivations doit être au moins égal à 2.", + "Disable window must be between 60 and 2592000 seconds.": "La fenêtre de désactivation doit être comprise entre 60 et 2592000 secondes.", "Minimum samples must be a non-negative integer.": "Minimum samples must be a non-negative integer.", - "Timeout ratio must be between 0 and 100 percent.": "Timeout ratio must be between 0 and 100 percent.", - "Disable duration must be between 0 and 604800 seconds.": "Disable duration must be between 0 and 604800 seconds.", - "Successful probe count must be between 1 and 10.": "Successful probe count must be between 1 and 10.", - "Penalty cooldown must be between 0 and 2592000 seconds.": "Penalty cooldown must be between 0 and 2592000 seconds.", - "Alert repeat interval must be between 0 and 2592000 seconds.": "Alert repeat interval must be between 0 and 2592000 seconds.", - "Recovery mode is invalid.": "Recovery mode is invalid.", - "Timeout count scope is invalid.": "Timeout count scope is invalid.", - "Channel health": "Channel health", - "Channel timeout governance": "Channel timeout governance", - "This view combines timeout alerts with the existing monitor_setting policy. Values are stored through the existing system option endpoint; no new database table or field is used.": "This view combines timeout alerts with the existing monitor_setting policy. Values are stored through the existing system option endpoint; no new database table or field is used.", - "Channel timeout alerts": "Channel timeout alerts", - "Priority demotions": "Priority demotions", - "Automatic disable alerts": "Automatic disable alerts", - "Timeout policy": "Timeout policy", - "Super administrators can change the existing monitor_setting values.": "Super administrators can change the existing monitor_setting values.", - "Only super administrators can change these values.": "Only super administrators can change these values.", - "Enable automatic priority demotion": "Enable automatic priority demotion", - "Reduce a channel priority after an eligible upstream timeout.": "Reduce a channel priority after an eligible upstream timeout.", - "Enable automatic channel disable": "Enable automatic channel disable", - "Disable a channel when its configured timeout count and ratio gates are met.": "Disable a channel when its configured timeout count and ratio gates are met.", - "Streaming first-result timeout": "Streaming first-result timeout", - "Seconds; 0 disables this threshold.": "Seconds; 0 disables this threshold.", - "Non-streaming response timeout": "Non-streaming response timeout", - "Priority deduction": "Priority deduction", - "Administrator-defined priority points per timeout.": "Administrator-defined priority points per timeout.", - "Penalty cooldown": "Penalty cooldown", - "Seconds before the same channel can be penalized again.": "Seconds before the same channel can be penalized again.", - "Alert repeat interval": "Alert repeat interval", - "Seconds between repeated notifications for one channel.": "Seconds between repeated notifications for one channel.", - "Disable after timeout count": "Disable after timeout count", - "Consecutive/window count used by the existing auto-disable policy.": "Consecutive/window count used by the existing auto-disable policy.", - "Auto-disable window": "Auto-disable window", - "Seconds used to evaluate timeout counts.": "Seconds used to evaluate timeout counts.", - "Minimum timeout samples": "Minimum timeout samples", - "Minimum observations before ratio-based disable is eligible.": "Minimum observations before ratio-based disable is eligible.", - "Timeout ratio threshold": "Timeout ratio threshold", - "Percentage threshold for the existing auto-disable policy.": "Percentage threshold for the existing auto-disable policy.", - "Auto-disable duration": "Auto-disable duration", - "Seconds a channel remains disabled before recovery checks.": "Seconds a channel remains disabled before recovery checks.", - "Successful recovery probes": "Successful recovery probes", - "Successful probes required before automatic recovery.": "Successful probes required before automatic recovery.", - "Expose channel demotion and disable transitions to the SmartOps alert stream.": "Expose channel demotion and disable transitions to the SmartOps alert stream.", - "Save timeout policy": "Save timeout policy", - "Channel alert stream": "Channel alert stream", - "Live channel timeout transitions from the existing SmartOps alert endpoint.": "Live channel timeout transitions from the existing SmartOps alert endpoint.", - "No active channel timeout alerts.": "No active channel timeout alerts.", - "Unknown channel": "Unknown channel", - "Channel {{id}}": "Channel {{id}}", - "This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts.": "This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts." + "Timeout ratio must be between 0 and 100 percent.": "Le ratio de dépassement doit être compris entre 0 et 100 %.", + "Disable duration must be between 0 and 604800 seconds.": "La durée de désactivation doit être comprise entre 0 et 604800 secondes.", + "Successful probe count must be between 1 and 10.": "Le nombre de sondes réussies doit être compris entre 1 et 10.", + "Penalty cooldown must be between 0 and 2592000 seconds.": "Le délai de refroidissement de la pénalité doit être compris entre 0 et 2592000 secondes.", + "Alert repeat interval must be between 0 and 2592000 seconds.": "L’intervalle de répétition des alertes doit être compris entre 0 et 2592000 secondes.", + "Recovery mode is invalid.": "Le mode de récupération est invalide.", + "Timeout count scope is invalid.": "La portée du comptage des dépassements est invalide.", + "Channel health": "Santé du canal", + "Channel timeout governance": "Gouvernance des dépassements des canaux", + "This view combines timeout alerts with the existing monitor_setting policy. Values are stored through the existing system option endpoint; no new database table or field is used.": "Cette vue combine les alertes de dépassement avec la politique monitor_setting existante. Les valeurs sont enregistrées via l’endpoint d’options système existant ; aucune nouvelle table ni aucun nouveau champ de base de données n’est utilisé.", + "Channel timeout alerts": "Alertes de dépassement des canaux", + "Priority demotions": "Rétrogradations de priorité", + "Automatic disable alerts": "Alertes de désactivation automatique", + "Timeout policy": "Politique de dépassement", + "Super administrators can change the existing monitor_setting values.": "Les super-administrateurs peuvent modifier les valeurs monitor_setting existantes.", + "Only super administrators can change these values.": "Seuls les super-administrateurs peuvent modifier ces valeurs.", + "Enable automatic priority demotion": "Activer la rétrogradation automatique de priorité", + "Reduce a channel priority after an eligible upstream timeout.": "Réduire la priorité d’un canal après un dépassement amont éligible.", + "Enable automatic channel disable": "Activer la désactivation automatique des canaux", + "Disable a channel when its configured timeout count and ratio gates are met.": "Désactiver un canal lorsque ses seuils configurés de nombre et de ratio de dépassements sont atteints.", + "Streaming first-result timeout": "Délai du premier résultat en streaming", + "Seconds; 0 disables this threshold.": "Secondes ; 0 désactive ce seuil.", + "Non-streaming response timeout": "Délai de réponse hors streaming", + "Priority deduction": "Déduction de priorité", + "Administrator-defined priority points per timeout.": "Points de priorité définis par l’administrateur pour chaque dépassement.", + "Penalty cooldown": "Délai de refroidissement de la pénalité", + "Seconds before the same channel can be penalized again.": "Secondes avant que le même canal puisse être pénalisé à nouveau.", + "Alert repeat interval": "Intervalle de répétition des alertes", + "Seconds between repeated notifications for one channel.": "Secondes entre les notifications répétées pour un canal.", + "Disable after timeout count": "Désactivation après le nombre de dépassements", + "Consecutive/window count used by the existing auto-disable policy.": "Nombre consécutif ou dans la fenêtre utilisé par la politique de désactivation automatique existante.", + "Auto-disable window": "Fenêtre de désactivation automatique", + "Seconds used to evaluate timeout counts.": "Secondes utilisées pour évaluer les compteurs de dépassement.", + "Minimum timeout samples": "Nombre minimal d’échantillons de dépassement", + "Minimum observations before ratio-based disable is eligible.": "Observations minimales avant l’éligibilité d’une désactivation fondée sur le ratio.", + "Timeout ratio threshold": "Seuil du ratio de dépassement", + "Percentage threshold for the existing auto-disable policy.": "Seuil en pourcentage de la politique de désactivation automatique existante.", + "Auto-disable duration": "Durée de désactivation automatique", + "Seconds a channel remains disabled before recovery checks.": "Secondes pendant lesquelles un canal reste désactivé avant les vérifications de récupération.", + "Successful recovery probes": "Sondes de récupération réussies", + "Successful probes required before automatic recovery.": "Nombre de sondes réussies requises avant la récupération automatique.", + "Expose channel demotion and disable transitions to the SmartOps alert stream.": "Exposer les transitions de rétrogradation et de désactivation des canaux dans le flux d’alertes SmartOps.", + "Save timeout policy": "Enregistrer la politique de dépassement", + "Channel alert stream": "Flux d’alertes des canaux", + "Live channel timeout transitions from the existing SmartOps alert endpoint.": "Transitions de dépassement des canaux en direct depuis l’endpoint d’alertes SmartOps existant.", + "No active channel timeout alerts.": "Aucune alerte active de dépassement de canal.", + "Unknown channel": "Canal inconnu", + "Channel {{id}}": "Canal {{id}}", + "This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts.": "Ce processus ne signale actuellement aucune pression persistante sur l’hôte, aucun dépassement de canal ni aucune alerte de rapprochement de facturation." } } diff --git a/web/default/src/i18n/locales/ja.json b/web/default/src/i18n/locales/ja.json index 3441232b..56df2467 100644 --- a/web/default/src/i18n/locales/ja.json +++ b/web/default/src/i18n/locales/ja.json @@ -5747,84 +5747,84 @@ "Enter an exact final quota for each selected task. Each value is checked against its own reserved quota.": "選択した各タスクの正確な最終クォータを入力してください。各値はタスク固有の予約クォータと照合されます。", "Apply exact settlements": "正確な精算を適用", "Allowed range: 0 to {{quota}} ({{quotaPerUnit}} quota = $1).": "許可範囲：0 ～ {{quota}}（{{quotaPerUnit}} クォータ = $1）。", - "Automatic recovery mode": "Automatic recovery mode", - "Probe then automatic": "Probe then automatic", - "Manual recovery": "Manual recovery", - "Timeout count scope": "Timeout count scope", - "Same request mode": "Same request mode", - "Combined modes": "Combined modes", - "Count streaming and non-streaming timeouts separately or together.": "Count streaming and non-streaming timeouts separately or together.", - "Probe mode recovers after configured successful probes; manual mode requires an administrator.": "Probe mode recovers after configured successful probes; manual mode requires an administrator.", - "Channel recovery failed": "Channel recovery failed", - "Channel recovery requested": "Channel recovery requested", - "Recover channel": "Recover channel", - "We could not load the channel health policy.": "We could not load the channel health policy.", - "Critical": "Critical", - "Channel timeout notifications": "Channel timeout notifications", - "Delivery status": "Delivery status", - "Recovery mode": "Recovery mode", - "Automatic": "Automatic", - "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.", - "Extend penalty on repeated timeout": "Extend penalty on repeated timeout", - "Keep the current penalty deadline when repeated timeouts continue during an active penalty.": "Keep the current penalty deadline when repeated timeouts continue during an active penalty.", - "{{count}} priority point": "{{count}} priority point", - "{{count}} priority points": "{{count}} priority points", - "{{count}} timeout": "{{count}} timeout", - "{{count}} timeouts": "{{count}} timeouts", - "Streaming first-result timeout must be between 0 and 86400 seconds.": "Streaming first-result timeout must be between 0 and 86400 seconds.", - "Non-streaming response timeout must be between 0 and 86400 seconds.": "Non-streaming response timeout must be between 0 and 86400 seconds.", - "Priority deduction must be a non-negative integer.": "Priority deduction must be a non-negative integer.", - "Disable count must be at least 2.": "Disable count must be at least 2.", - "Disable window must be between 60 and 2592000 seconds.": "Disable window must be between 60 and 2592000 seconds.", + "Automatic recovery mode": "自動復旧モード", + "Probe then automatic": "プローブ後に自動復旧", + "Manual recovery": "手動復旧", + "Timeout count scope": "タイムアウト集計範囲", + "Same request mode": "同じリクエストモード", + "Combined modes": "ストリーミングと非ストリーミングを統合", + "Count streaming and non-streaming timeouts separately or together.": "ストリーミングと非ストリーミングのタイムアウトを個別または合算して集計します。", + "Probe mode recovers after configured successful probes; manual mode requires an administrator.": "プローブモードは設定回数の成功プローブ後に復旧し、手動モードでは管理者の操作が必要です。", + "Channel recovery failed": "チャネルの復旧に失敗しました", + "Channel recovery requested": "チャネルの復旧を要求しました", + "Recover channel": "チャネルを復旧", + "We could not load the channel health policy.": "チャネルヘルスポリシーを読み込めませんでした。", + "Critical": "重大", + "Channel timeout notifications": "チャネルタイムアウト通知", + "Delivery status": "配信状態", + "Recovery mode": "復旧モード", + "Automatic": "自動", + "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "自動復旧はクールダウン後にペナルティを解除し、手動復旧では管理者が操作するまでペナルティを維持します。", + "Extend penalty on repeated timeout": "タイムアウト再発時にペナルティを延長", + "Keep the current penalty deadline when repeated timeouts continue during an active penalty.": "ペナルティ中にタイムアウトが再発しても、現在のペナルティ期限を維持します。", + "{{count}} priority point": "優先度ポイント {{count}}", + "{{count}} priority points": "優先度ポイント {{count}}", + "{{count}} timeout": "タイムアウト {{count}} 回", + "{{count}} timeouts": "タイムアウト {{count}} 回", + "Streaming first-result timeout must be between 0 and 86400 seconds.": "ストリーミング初回結果タイムアウトは 0～86400 秒で指定してください。", + "Non-streaming response timeout must be between 0 and 86400 seconds.": "非ストリーミング応答タイムアウトは 0～86400 秒で指定してください。", + "Priority deduction must be a non-negative integer.": "優先度の減点は 0 以上の整数で指定してください。", + "Disable count must be at least 2.": "無効化回数は 2 以上で指定してください。", + "Disable window must be between 60 and 2592000 seconds.": "無効化ウィンドウは 60～2592000 秒で指定してください。", "Minimum samples must be a non-negative integer.": "Minimum samples must be a non-negative integer.", - "Timeout ratio must be between 0 and 100 percent.": "Timeout ratio must be between 0 and 100 percent.", - "Disable duration must be between 0 and 604800 seconds.": "Disable duration must be between 0 and 604800 seconds.", - "Successful probe count must be between 1 and 10.": "Successful probe count must be between 1 and 10.", - "Penalty cooldown must be between 0 and 2592000 seconds.": "Penalty cooldown must be between 0 and 2592000 seconds.", - "Alert repeat interval must be between 0 and 2592000 seconds.": "Alert repeat interval must be between 0 and 2592000 seconds.", - "Recovery mode is invalid.": "Recovery mode is invalid.", - "Timeout count scope is invalid.": "Timeout count scope is invalid.", - "Channel health": "Channel health", - "Channel timeout governance": "Channel timeout governance", - "This view combines timeout alerts with the existing monitor_setting policy. Values are stored through the existing system option endpoint; no new database table or field is used.": "This view combines timeout alerts with the existing monitor_setting policy. Values are stored through the existing system option endpoint; no new database table or field is used.", - "Channel timeout alerts": "Channel timeout alerts", - "Priority demotions": "Priority demotions", - "Automatic disable alerts": "Automatic disable alerts", - "Timeout policy": "Timeout policy", - "Super administrators can change the existing monitor_setting values.": "Super administrators can change the existing monitor_setting values.", - "Only super administrators can change these values.": "Only super administrators can change these values.", - "Enable automatic priority demotion": "Enable automatic priority demotion", - "Reduce a channel priority after an eligible upstream timeout.": "Reduce a channel priority after an eligible upstream timeout.", - "Enable automatic channel disable": "Enable automatic channel disable", - "Disable a channel when its configured timeout count and ratio gates are met.": "Disable a channel when its configured timeout count and ratio gates are met.", - "Streaming first-result timeout": "Streaming first-result timeout", - "Seconds; 0 disables this threshold.": "Seconds; 0 disables this threshold.", - "Non-streaming response timeout": "Non-streaming response timeout", - "Priority deduction": "Priority deduction", - "Administrator-defined priority points per timeout.": "Administrator-defined priority points per timeout.", - "Penalty cooldown": "Penalty cooldown", - "Seconds before the same channel can be penalized again.": "Seconds before the same channel can be penalized again.", - "Alert repeat interval": "Alert repeat interval", - "Seconds between repeated notifications for one channel.": "Seconds between repeated notifications for one channel.", - "Disable after timeout count": "Disable after timeout count", - "Consecutive/window count used by the existing auto-disable policy.": "Consecutive/window count used by the existing auto-disable policy.", - "Auto-disable window": "Auto-disable window", - "Seconds used to evaluate timeout counts.": "Seconds used to evaluate timeout counts.", - "Minimum timeout samples": "Minimum timeout samples", - "Minimum observations before ratio-based disable is eligible.": "Minimum observations before ratio-based disable is eligible.", - "Timeout ratio threshold": "Timeout ratio threshold", - "Percentage threshold for the existing auto-disable policy.": "Percentage threshold for the existing auto-disable policy.", - "Auto-disable duration": "Auto-disable duration", - "Seconds a channel remains disabled before recovery checks.": "Seconds a channel remains disabled before recovery checks.", - "Successful recovery probes": "Successful recovery probes", - "Successful probes required before automatic recovery.": "Successful probes required before automatic recovery.", - "Expose channel demotion and disable transitions to the SmartOps alert stream.": "Expose channel demotion and disable transitions to the SmartOps alert stream.", - "Save timeout policy": "Save timeout policy", - "Channel alert stream": "Channel alert stream", - "Live channel timeout transitions from the existing SmartOps alert endpoint.": "Live channel timeout transitions from the existing SmartOps alert endpoint.", - "No active channel timeout alerts.": "No active channel timeout alerts.", - "Unknown channel": "Unknown channel", - "Channel {{id}}": "Channel {{id}}", - "This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts.": "This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts." + "Timeout ratio must be between 0 and 100 percent.": "タイムアウト比率は 0～100 パーセントで指定してください。", + "Disable duration must be between 0 and 604800 seconds.": "無効化期間は 0～604800 秒で指定してください。", + "Successful probe count must be between 1 and 10.": "成功プローブ数は 1～10 で指定してください。", + "Penalty cooldown must be between 0 and 2592000 seconds.": "ペナルティのクールダウンは 0～2592000 秒で指定してください。", + "Alert repeat interval must be between 0 and 2592000 seconds.": "アラート再通知間隔は 0～2592000 秒で指定してください。", + "Recovery mode is invalid.": "復旧モードが無効です。", + "Timeout count scope is invalid.": "タイムアウト集計範囲が無効です。", + "Channel health": "チャネルヘルス", + "Channel timeout governance": "チャネルタイムアウト管理", + "This view combines timeout alerts with the existing monitor_setting policy. Values are stored through the existing system option endpoint; no new database table or field is used.": "この画面ではタイムアウトアラートと既存の monitor_setting ポリシーをまとめて管理します。値は既存のシステムオプション API に保存され、新しいデータベースのテーブルやフィールドは使用しません。", + "Channel timeout alerts": "チャネルタイムアウトアラート", + "Priority demotions": "優先度低下", + "Automatic disable alerts": "自動無効化アラート", + "Timeout policy": "タイムアウトポリシー", + "Super administrators can change the existing monitor_setting values.": "スーパー管理者は既存の monitor_setting 値を変更できます。", + "Only super administrators can change these values.": "これらの値を変更できるのはスーパー管理者だけです。", + "Enable automatic priority demotion": "優先度の自動低下を有効化", + "Reduce a channel priority after an eligible upstream timeout.": "対象となる上流タイムアウト後にチャネルの優先度を下げます。", + "Enable automatic channel disable": "チャネルの自動無効化を有効化", + "Disable a channel when its configured timeout count and ratio gates are met.": "設定したタイムアウト回数と比率の条件を満たしたときにチャネルを無効化します。", + "Streaming first-result timeout": "ストリーミング初回結果タイムアウト", + "Seconds; 0 disables this threshold.": "秒。0 はこのしきい値を無効にします。", + "Non-streaming response timeout": "非ストリーミング応答タイムアウト", + "Priority deduction": "優先度の減点", + "Administrator-defined priority points per timeout.": "タイムアウトごとに管理者が指定する優先度ポイントです。", + "Penalty cooldown": "ペナルティのクールダウン", + "Seconds before the same channel can be penalized again.": "同じチャネルに再びペナルティを適用できるまでの秒数です。", + "Alert repeat interval": "アラート再通知間隔", + "Seconds between repeated notifications for one channel.": "1 チャネルへの通知を繰り返す間隔（秒）です。", + "Disable after timeout count": "タイムアウト回数で無効化", + "Consecutive/window count used by the existing auto-disable policy.": "既存の自動無効化ポリシーで使用する連続または期間内の回数です。", + "Auto-disable window": "自動無効化ウィンドウ", + "Seconds used to evaluate timeout counts.": "タイムアウト回数を評価する期間（秒）です。", + "Minimum timeout samples": "タイムアウト最小サンプル数", + "Minimum observations before ratio-based disable is eligible.": "比率による無効化を判定できるようになるまでに必要な最小観測数です。", + "Timeout ratio threshold": "タイムアウト比率しきい値", + "Percentage threshold for the existing auto-disable policy.": "既存の自動無効化ポリシーで使用する比率のしきい値です。", + "Auto-disable duration": "自動無効化期間", + "Seconds a channel remains disabled before recovery checks.": "復旧チェックまでチャネルを無効にする秒数です。", + "Successful recovery probes": "復旧成功プローブ数", + "Successful probes required before automatic recovery.": "自動復旧に必要な成功プローブ数です。", + "Expose channel demotion and disable transitions to the SmartOps alert stream.": "チャネルの優先度低下と無効化の遷移を SmartOps アラートストリームに表示します。", + "Save timeout policy": "タイムアウトポリシーを保存", + "Channel alert stream": "チャネルアラートストリーム", + "Live channel timeout transitions from the existing SmartOps alert endpoint.": "既存の SmartOps アラート API から取得したチャネルタイムアウトの最新遷移です。", + "No active channel timeout alerts.": "アクティブなチャネルタイムアウトアラートはありません。", + "Unknown channel": "不明なチャネル", + "Channel {{id}}": "チャネル {{id}}", + "This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts.": "このプロセスでは現在、継続的なホスト負荷、チャネルタイムアウト、請求照合のアラートは報告されていません。" } } diff --git a/web/default/src/i18n/locales/ru.json b/web/default/src/i18n/locales/ru.json index ce3a216b..6e3f7613 100644 --- a/web/default/src/i18n/locales/ru.json +++ b/web/default/src/i18n/locales/ru.json @@ -5755,84 +5755,84 @@ "This will settle {{displayCount}} selected MiniMax-H3 task(s) with an explicit final quota of 0 and refund the unused reservation. Continue only after verifying the provider result._many": "Будут выполнены расчёты для {{displayCount}} выбранных задач MiniMax-H3 с явной итоговой квотой 0 и возвращён неиспользованный резерв. Продолжайте только после проверки результата провайдера.", "Enter exact quotas ({{count}})_few": "Ввести точные квоты ({{count}})", "Enter exact quotas ({{count}})_many": "Ввести точные квоты ({{count}})", - "Automatic recovery mode": "Automatic recovery mode", - "Probe then automatic": "Probe then automatic", - "Manual recovery": "Manual recovery", - "Timeout count scope": "Timeout count scope", - "Same request mode": "Same request mode", - "Combined modes": "Combined modes", - "Count streaming and non-streaming timeouts separately or together.": "Count streaming and non-streaming timeouts separately or together.", - "Probe mode recovers after configured successful probes; manual mode requires an administrator.": "Probe mode recovers after configured successful probes; manual mode requires an administrator.", - "Channel recovery failed": "Channel recovery failed", - "Channel recovery requested": "Channel recovery requested", - "Recover channel": "Recover channel", - "We could not load the channel health policy.": "We could not load the channel health policy.", - "Critical": "Critical", - "Channel timeout notifications": "Channel timeout notifications", - "Delivery status": "Delivery status", - "Recovery mode": "Recovery mode", - "Automatic": "Automatic", - "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.", - "Extend penalty on repeated timeout": "Extend penalty on repeated timeout", - "Keep the current penalty deadline when repeated timeouts continue during an active penalty.": "Keep the current penalty deadline when repeated timeouts continue during an active penalty.", - "{{count}} priority point": "{{count}} priority point", - "{{count}} priority points": "{{count}} priority points", - "{{count}} timeout": "{{count}} timeout", - "{{count}} timeouts": "{{count}} timeouts", - "Streaming first-result timeout must be between 0 and 86400 seconds.": "Streaming first-result timeout must be between 0 and 86400 seconds.", - "Non-streaming response timeout must be between 0 and 86400 seconds.": "Non-streaming response timeout must be between 0 and 86400 seconds.", - "Priority deduction must be a non-negative integer.": "Priority deduction must be a non-negative integer.", - "Disable count must be at least 2.": "Disable count must be at least 2.", - "Disable window must be between 60 and 2592000 seconds.": "Disable window must be between 60 and 2592000 seconds.", + "Automatic recovery mode": "Режим автоматического восстановления", + "Probe then automatic": "Проверка, затем автоматическое восстановление", + "Manual recovery": "Ручное восстановление", + "Timeout count scope": "Область подсчёта таймаутов", + "Same request mode": "Один режим запроса", + "Combined modes": "Объединённые режимы", + "Count streaming and non-streaming timeouts separately or together.": "Считать таймауты потоковых и непотоковых запросов отдельно или вместе.", + "Probe mode recovers after configured successful probes; manual mode requires an administrator.": "В режиме проверки восстановление выполняется после заданного числа успешных проверок; ручной режим требует действий администратора.", + "Channel recovery failed": "Не удалось восстановить канал", + "Channel recovery requested": "Запрошено восстановление канала", + "Recover channel": "Восстановить канал", + "We could not load the channel health policy.": "Не удалось загрузить политику состояния канала.", + "Critical": "Критический", + "Channel timeout notifications": "Уведомления о таймаутах канала", + "Delivery status": "Состояние доставки", + "Recovery mode": "Режим восстановления", + "Automatic": "Автоматический", + "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "Автоматическое восстановление снимает штраф после периода охлаждения; при ручном восстановлении штраф сохраняется до действий администратора.", + "Extend penalty on repeated timeout": "Продлевать штраф при повторном таймауте", + "Keep the current penalty deadline when repeated timeouts continue during an active penalty.": "Сохранять текущий срок штрафа, если повторные таймауты происходят во время активного штрафа.", + "{{count}} priority point": "{{count}} балл приоритета", + "{{count}} priority points": "{{count}} баллов приоритета", + "{{count}} timeout": "{{count}} таймаут", + "{{count}} timeouts": "{{count}} таймаутов", + "Streaming first-result timeout must be between 0 and 86400 seconds.": "Таймаут первого результата потокового запроса должен быть от 0 до 86400 секунд.", + "Non-streaming response timeout must be between 0 and 86400 seconds.": "Таймаут ответа непотокового запроса должен быть от 0 до 86400 секунд.", + "Priority deduction must be a non-negative integer.": "Снижение приоритета должно быть неотрицательным целым числом.", + "Disable count must be at least 2.": "Число отключений должно быть не меньше 2.", + "Disable window must be between 60 and 2592000 seconds.": "Окно отключения должно быть от 60 до 2592000 секунд.", "Minimum samples must be a non-negative integer.": "Minimum samples must be a non-negative integer.", - "Timeout ratio must be between 0 and 100 percent.": "Timeout ratio must be between 0 and 100 percent.", - "Disable duration must be between 0 and 604800 seconds.": "Disable duration must be between 0 and 604800 seconds.", - "Successful probe count must be between 1 and 10.": "Successful probe count must be between 1 and 10.", - "Penalty cooldown must be between 0 and 2592000 seconds.": "Penalty cooldown must be between 0 and 2592000 seconds.", - "Alert repeat interval must be between 0 and 2592000 seconds.": "Alert repeat interval must be between 0 and 2592000 seconds.", - "Recovery mode is invalid.": "Recovery mode is invalid.", - "Timeout count scope is invalid.": "Timeout count scope is invalid.", - "Channel health": "Channel health", - "Channel timeout governance": "Channel timeout governance", - "This view combines timeout alerts with the existing monitor_setting policy. Values are stored through the existing system option endpoint; no new database table or field is used.": "This view combines timeout alerts with the existing monitor_setting policy. Values are stored through the existing system option endpoint; no new database table or field is used.", - "Channel timeout alerts": "Channel timeout alerts", - "Priority demotions": "Priority demotions", - "Automatic disable alerts": "Automatic disable alerts", - "Timeout policy": "Timeout policy", - "Super administrators can change the existing monitor_setting values.": "Super administrators can change the existing monitor_setting values.", - "Only super administrators can change these values.": "Only super administrators can change these values.", - "Enable automatic priority demotion": "Enable automatic priority demotion", - "Reduce a channel priority after an eligible upstream timeout.": "Reduce a channel priority after an eligible upstream timeout.", - "Enable automatic channel disable": "Enable automatic channel disable", - "Disable a channel when its configured timeout count and ratio gates are met.": "Disable a channel when its configured timeout count and ratio gates are met.", - "Streaming first-result timeout": "Streaming first-result timeout", - "Seconds; 0 disables this threshold.": "Seconds; 0 disables this threshold.", - "Non-streaming response timeout": "Non-streaming response timeout", - "Priority deduction": "Priority deduction", - "Administrator-defined priority points per timeout.": "Administrator-defined priority points per timeout.", - "Penalty cooldown": "Penalty cooldown", - "Seconds before the same channel can be penalized again.": "Seconds before the same channel can be penalized again.", - "Alert repeat interval": "Alert repeat interval", - "Seconds between repeated notifications for one channel.": "Seconds between repeated notifications for one channel.", - "Disable after timeout count": "Disable after timeout count", - "Consecutive/window count used by the existing auto-disable policy.": "Consecutive/window count used by the existing auto-disable policy.", - "Auto-disable window": "Auto-disable window", - "Seconds used to evaluate timeout counts.": "Seconds used to evaluate timeout counts.", - "Minimum timeout samples": "Minimum timeout samples", - "Minimum observations before ratio-based disable is eligible.": "Minimum observations before ratio-based disable is eligible.", - "Timeout ratio threshold": "Timeout ratio threshold", - "Percentage threshold for the existing auto-disable policy.": "Percentage threshold for the existing auto-disable policy.", - "Auto-disable duration": "Auto-disable duration", - "Seconds a channel remains disabled before recovery checks.": "Seconds a channel remains disabled before recovery checks.", - "Successful recovery probes": "Successful recovery probes", - "Successful probes required before automatic recovery.": "Successful probes required before automatic recovery.", - "Expose channel demotion and disable transitions to the SmartOps alert stream.": "Expose channel demotion and disable transitions to the SmartOps alert stream.", - "Save timeout policy": "Save timeout policy", - "Channel alert stream": "Channel alert stream", - "Live channel timeout transitions from the existing SmartOps alert endpoint.": "Live channel timeout transitions from the existing SmartOps alert endpoint.", - "No active channel timeout alerts.": "No active channel timeout alerts.", - "Unknown channel": "Unknown channel", - "Channel {{id}}": "Channel {{id}}", - "This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts.": "This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts." + "Timeout ratio must be between 0 and 100 percent.": "Доля таймаутов должна быть от 0 до 100 процентов.", + "Disable duration must be between 0 and 604800 seconds.": "Длительность отключения должна быть от 0 до 604800 секунд.", + "Successful probe count must be between 1 and 10.": "Число успешных проверок должно быть от 1 до 10.", + "Penalty cooldown must be between 0 and 2592000 seconds.": "Период охлаждения штрафа должен быть от 0 до 2592000 секунд.", + "Alert repeat interval must be between 0 and 2592000 seconds.": "Интервал повторения оповещений должен быть от 0 до 2592000 секунд.", + "Recovery mode is invalid.": "Недопустимый режим восстановления.", + "Timeout count scope is invalid.": "Недопустимая область подсчёта таймаутов.", + "Channel health": "Состояние канала", + "Channel timeout governance": "Управление таймаутами каналов", + "This view combines timeout alerts with the existing monitor_setting policy. Values are stored through the existing system option endpoint; no new database table or field is used.": "На этой странице объединены оповещения о таймаутах и существующая политика monitor_setting. Значения сохраняются через существующий API системных параметров; новые таблицы и поля базы данных не используются.", + "Channel timeout alerts": "Оповещения о таймаутах каналов", + "Priority demotions": "Снижение приоритета", + "Automatic disable alerts": "Оповещения об автоотключении", + "Timeout policy": "Политика таймаутов", + "Super administrators can change the existing monitor_setting values.": "Суперадминистраторы могут изменять существующие значения monitor_setting.", + "Only super administrators can change these values.": "Изменять эти значения могут только суперадминистраторы.", + "Enable automatic priority demotion": "Включить автоматическое снижение приоритета", + "Reduce a channel priority after an eligible upstream timeout.": "Снижать приоритет канала после подходящего таймаута upstream.", + "Enable automatic channel disable": "Включить автоматическое отключение каналов", + "Disable a channel when its configured timeout count and ratio gates are met.": "Отключать канал при достижении заданных порогов числа и доли таймаутов.", + "Streaming first-result timeout": "Таймаут первого результата потокового запроса", + "Seconds; 0 disables this threshold.": "Секунды; 0 отключает этот порог.", + "Non-streaming response timeout": "Таймаут ответа непотокового запроса", + "Priority deduction": "Снижение приоритета", + "Administrator-defined priority points per timeout.": "Количество баллов приоритета на таймаут, заданное администратором.", + "Penalty cooldown": "Период охлаждения штрафа", + "Seconds before the same channel can be penalized again.": "Количество секунд до повторного штрафа для того же канала.", + "Alert repeat interval": "Интервал повторения оповещений", + "Seconds between repeated notifications for one channel.": "Количество секунд между повторными уведомлениями для одного канала.", + "Disable after timeout count": "Отключение после числа таймаутов", + "Consecutive/window count used by the existing auto-disable policy.": "Последовательное число или число в окне, используемое существующей политикой автоотключения.", + "Auto-disable window": "Окно автоотключения", + "Seconds used to evaluate timeout counts.": "Количество секунд для оценки числа таймаутов.", + "Minimum timeout samples": "Минимум образцов таймаутов", + "Minimum observations before ratio-based disable is eligible.": "Минимальное число наблюдений до возможности отключения по доле таймаутов.", + "Timeout ratio threshold": "Порог доли таймаутов", + "Percentage threshold for the existing auto-disable policy.": "Процентный порог существующей политики автоотключения.", + "Auto-disable duration": "Длительность автоотключения", + "Seconds a channel remains disabled before recovery checks.": "Количество секунд, в течение которых канал отключён до проверок восстановления.", + "Successful recovery probes": "Успешные проверки восстановления", + "Successful probes required before automatic recovery.": "Число успешных проверок, необходимое для автоматического восстановления.", + "Expose channel demotion and disable transitions to the SmartOps alert stream.": "Показывать переходы снижения приоритета и отключения каналов в потоке оповещений SmartOps.", + "Save timeout policy": "Сохранить политику таймаутов", + "Channel alert stream": "Поток оповещений каналов", + "Live channel timeout transitions from the existing SmartOps alert endpoint.": "Текущие переходы таймаутов каналов из существующего API оповещений SmartOps.", + "No active channel timeout alerts.": "Нет активных оповещений о таймаутах каналов.", + "Unknown channel": "Неизвестный канал", + "Channel {{id}}": "Канал {{id}}", + "This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts.": "Этот процесс сейчас не сообщает о длительной нагрузке на хост, таймаутах каналов или оповещениях сверки биллинга." } } diff --git a/web/default/src/i18n/locales/vi.json b/web/default/src/i18n/locales/vi.json index 37bf3d27..58017fa1 100644 --- a/web/default/src/i18n/locales/vi.json +++ b/web/default/src/i18n/locales/vi.json @@ -5747,84 +5747,84 @@ "Enter an exact final quota for each selected task. Each value is checked against its own reserved quota.": "Nhập hạn mức cuối chính xác cho từng tác vụ đã chọn. Mỗi giá trị sẽ được kiểm tra theo hạn mức đã giữ riêng của tác vụ đó.", "Apply exact settlements": "Áp dụng các quyết toán chính xác", "Allowed range: 0 to {{quota}} ({{quotaPerUnit}} quota = $1).": "Phạm vi cho phép: từ 0 đến {{quota}} ({{quotaPerUnit}} đơn vị hạn mức = $1).", - "Automatic recovery mode": "Automatic recovery mode", - "Probe then automatic": "Probe then automatic", - "Manual recovery": "Manual recovery", - "Timeout count scope": "Timeout count scope", - "Same request mode": "Same request mode", - "Combined modes": "Combined modes", - "Count streaming and non-streaming timeouts separately or together.": "Count streaming and non-streaming timeouts separately or together.", - "Probe mode recovers after configured successful probes; manual mode requires an administrator.": "Probe mode recovers after configured successful probes; manual mode requires an administrator.", - "Channel recovery failed": "Channel recovery failed", - "Channel recovery requested": "Channel recovery requested", - "Recover channel": "Recover channel", - "We could not load the channel health policy.": "We could not load the channel health policy.", - "Critical": "Critical", - "Channel timeout notifications": "Channel timeout notifications", - "Delivery status": "Delivery status", - "Recovery mode": "Recovery mode", - "Automatic": "Automatic", - "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.", - "Extend penalty on repeated timeout": "Extend penalty on repeated timeout", - "Keep the current penalty deadline when repeated timeouts continue during an active penalty.": "Keep the current penalty deadline when repeated timeouts continue during an active penalty.", - "{{count}} priority point": "{{count}} priority point", - "{{count}} priority points": "{{count}} priority points", - "{{count}} timeout": "{{count}} timeout", - "{{count}} timeouts": "{{count}} timeouts", - "Streaming first-result timeout must be between 0 and 86400 seconds.": "Streaming first-result timeout must be between 0 and 86400 seconds.", - "Non-streaming response timeout must be between 0 and 86400 seconds.": "Non-streaming response timeout must be between 0 and 86400 seconds.", - "Priority deduction must be a non-negative integer.": "Priority deduction must be a non-negative integer.", - "Disable count must be at least 2.": "Disable count must be at least 2.", - "Disable window must be between 60 and 2592000 seconds.": "Disable window must be between 60 and 2592000 seconds.", + "Automatic recovery mode": "Chế độ khôi phục tự động", + "Probe then automatic": "Thăm dò rồi tự động khôi phục", + "Manual recovery": "Khôi phục thủ công", + "Timeout count scope": "Phạm vi đếm thời gian chờ", + "Same request mode": "Cùng chế độ yêu cầu", + "Combined modes": "Gộp các chế độ", + "Count streaming and non-streaming timeouts separately or together.": "Đếm thời gian chờ của luồng và không luồng riêng biệt hoặc gộp chung.", + "Probe mode recovers after configured successful probes; manual mode requires an administrator.": "Chế độ thăm dò khôi phục sau số lần thăm dò thành công đã cấu hình; chế độ thủ công cần quản trị viên thao tác.", + "Channel recovery failed": "Khôi phục kênh thất bại", + "Channel recovery requested": "Đã yêu cầu khôi phục kênh", + "Recover channel": "Khôi phục kênh", + "We could not load the channel health policy.": "Không thể tải chính sách tình trạng kênh.", + "Critical": "Nghiêm trọng", + "Channel timeout notifications": "Thông báo thời gian chờ của kênh", + "Delivery status": "Trạng thái gửi", + "Recovery mode": "Chế độ khôi phục", + "Automatic": "Tự động", + "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "Khôi phục tự động xóa mức phạt sau thời gian nguội; khôi phục thủ công giữ mức phạt cho đến khi quản trị viên thao tác.", + "Extend penalty on repeated timeout": "Gia hạn mức phạt khi thời gian chờ lặp lại", + "Keep the current penalty deadline when repeated timeouts continue during an active penalty.": "Giữ hạn mức phạt hiện tại khi thời gian chờ tiếp tục lặp lại trong lúc đang bị phạt.", + "{{count}} priority point": "{{count}} điểm ưu tiên", + "{{count}} priority points": "{{count}} điểm ưu tiên", + "{{count}} timeout": "{{count}} lần chờ quá hạn", + "{{count}} timeouts": "{{count}} lần chờ quá hạn", + "Streaming first-result timeout must be between 0 and 86400 seconds.": "Thời gian chờ kết quả đầu tiên của luồng phải từ 0 đến 86400 giây.", + "Non-streaming response timeout must be between 0 and 86400 seconds.": "Thời gian chờ phản hồi không luồng phải từ 0 đến 86400 giây.", + "Priority deduction must be a non-negative integer.": "Mức trừ ưu tiên phải là số nguyên không âm.", + "Disable count must be at least 2.": "Số lần tắt phải ít nhất là 2.", + "Disable window must be between 60 and 2592000 seconds.": "Cửa sổ tắt phải từ 60 đến 2592000 giây.", "Minimum samples must be a non-negative integer.": "Minimum samples must be a non-negative integer.", - "Timeout ratio must be between 0 and 100 percent.": "Timeout ratio must be between 0 and 100 percent.", - "Disable duration must be between 0 and 604800 seconds.": "Disable duration must be between 0 and 604800 seconds.", - "Successful probe count must be between 1 and 10.": "Successful probe count must be between 1 and 10.", - "Penalty cooldown must be between 0 and 2592000 seconds.": "Penalty cooldown must be between 0 and 2592000 seconds.", - "Alert repeat interval must be between 0 and 2592000 seconds.": "Alert repeat interval must be between 0 and 2592000 seconds.", - "Recovery mode is invalid.": "Recovery mode is invalid.", - "Timeout count scope is invalid.": "Timeout count scope is invalid.", - "Channel health": "Channel health", - "Channel timeout governance": "Channel timeout governance", - "This view combines timeout alerts with the existing monitor_setting policy. Values are stored through the existing system option endpoint; no new database table or field is used.": "This view combines timeout alerts with the existing monitor_setting policy. Values are stored through the existing system option endpoint; no new database table or field is used.", - "Channel timeout alerts": "Channel timeout alerts", - "Priority demotions": "Priority demotions", - "Automatic disable alerts": "Automatic disable alerts", - "Timeout policy": "Timeout policy", - "Super administrators can change the existing monitor_setting values.": "Super administrators can change the existing monitor_setting values.", - "Only super administrators can change these values.": "Only super administrators can change these values.", - "Enable automatic priority demotion": "Enable automatic priority demotion", - "Reduce a channel priority after an eligible upstream timeout.": "Reduce a channel priority after an eligible upstream timeout.", - "Enable automatic channel disable": "Enable automatic channel disable", - "Disable a channel when its configured timeout count and ratio gates are met.": "Disable a channel when its configured timeout count and ratio gates are met.", - "Streaming first-result timeout": "Streaming first-result timeout", - "Seconds; 0 disables this threshold.": "Seconds; 0 disables this threshold.", - "Non-streaming response timeout": "Non-streaming response timeout", - "Priority deduction": "Priority deduction", - "Administrator-defined priority points per timeout.": "Administrator-defined priority points per timeout.", - "Penalty cooldown": "Penalty cooldown", - "Seconds before the same channel can be penalized again.": "Seconds before the same channel can be penalized again.", - "Alert repeat interval": "Alert repeat interval", - "Seconds between repeated notifications for one channel.": "Seconds between repeated notifications for one channel.", - "Disable after timeout count": "Disable after timeout count", - "Consecutive/window count used by the existing auto-disable policy.": "Consecutive/window count used by the existing auto-disable policy.", - "Auto-disable window": "Auto-disable window", - "Seconds used to evaluate timeout counts.": "Seconds used to evaluate timeout counts.", - "Minimum timeout samples": "Minimum timeout samples", - "Minimum observations before ratio-based disable is eligible.": "Minimum observations before ratio-based disable is eligible.", - "Timeout ratio threshold": "Timeout ratio threshold", - "Percentage threshold for the existing auto-disable policy.": "Percentage threshold for the existing auto-disable policy.", - "Auto-disable duration": "Auto-disable duration", - "Seconds a channel remains disabled before recovery checks.": "Seconds a channel remains disabled before recovery checks.", - "Successful recovery probes": "Successful recovery probes", - "Successful probes required before automatic recovery.": "Successful probes required before automatic recovery.", - "Expose channel demotion and disable transitions to the SmartOps alert stream.": "Expose channel demotion and disable transitions to the SmartOps alert stream.", - "Save timeout policy": "Save timeout policy", - "Channel alert stream": "Channel alert stream", - "Live channel timeout transitions from the existing SmartOps alert endpoint.": "Live channel timeout transitions from the existing SmartOps alert endpoint.", - "No active channel timeout alerts.": "No active channel timeout alerts.", - "Unknown channel": "Unknown channel", - "Channel {{id}}": "Channel {{id}}", - "This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts.": "This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts." + "Timeout ratio must be between 0 and 100 percent.": "Tỷ lệ chờ quá hạn phải từ 0 đến 100 phần trăm.", + "Disable duration must be between 0 and 604800 seconds.": "Thời lượng tắt phải từ 0 đến 604800 giây.", + "Successful probe count must be between 1 and 10.": "Số lần thăm dò thành công phải từ 1 đến 10.", + "Penalty cooldown must be between 0 and 2592000 seconds.": "Thời gian nguội của mức phạt phải từ 0 đến 2592000 giây.", + "Alert repeat interval must be between 0 and 2592000 seconds.": "Khoảng lặp cảnh báo phải từ 0 đến 2592000 giây.", + "Recovery mode is invalid.": "Chế độ khôi phục không hợp lệ.", + "Timeout count scope is invalid.": "Phạm vi đếm thời gian chờ không hợp lệ.", + "Channel health": "Tình trạng kênh", + "Channel timeout governance": "Quản trị thời gian chờ của kênh", + "This view combines timeout alerts with the existing monitor_setting policy. Values are stored through the existing system option endpoint; no new database table or field is used.": "Màn hình này kết hợp cảnh báo thời gian chờ với chính sách monitor_setting hiện có. Giá trị được lưu qua API tùy chọn hệ thống hiện có; không sử dụng bảng hoặc trường cơ sở dữ liệu mới.", + "Channel timeout alerts": "Cảnh báo thời gian chờ của kênh", + "Priority demotions": "Hạ mức ưu tiên", + "Automatic disable alerts": "Cảnh báo tự động tắt", + "Timeout policy": "Chính sách thời gian chờ", + "Super administrators can change the existing monitor_setting values.": "Quản trị viên cấp cao có thể thay đổi các giá trị monitor_setting hiện có.", + "Only super administrators can change these values.": "Chỉ quản trị viên cấp cao mới có thể thay đổi các giá trị này.", + "Enable automatic priority demotion": "Bật tự động hạ mức ưu tiên", + "Reduce a channel priority after an eligible upstream timeout.": "Giảm ưu tiên của kênh sau khi upstream xảy ra thời gian chờ đủ điều kiện.", + "Enable automatic channel disable": "Bật tự động tắt kênh", + "Disable a channel when its configured timeout count and ratio gates are met.": "Tắt kênh khi đạt các ngưỡng số lần và tỷ lệ chờ quá hạn đã cấu hình.", + "Streaming first-result timeout": "Thời gian chờ kết quả đầu tiên của luồng", + "Seconds; 0 disables this threshold.": "Giây; 0 tắt ngưỡng này.", + "Non-streaming response timeout": "Thời gian chờ phản hồi không luồng", + "Priority deduction": "Mức trừ ưu tiên", + "Administrator-defined priority points per timeout.": "Số điểm ưu tiên do quản trị viên đặt cho mỗi lần chờ quá hạn.", + "Penalty cooldown": "Thời gian nguội của mức phạt", + "Seconds before the same channel can be penalized again.": "Số giây trước khi cùng một kênh có thể bị phạt lại.", + "Alert repeat interval": "Khoảng lặp cảnh báo", + "Seconds between repeated notifications for one channel.": "Số giây giữa các thông báo lặp lại cho một kênh.", + "Disable after timeout count": "Tắt sau số lần chờ quá hạn", + "Consecutive/window count used by the existing auto-disable policy.": "Số lần liên tiếp hoặc trong cửa sổ được dùng bởi chính sách tự động tắt hiện có.", + "Auto-disable window": "Cửa sổ tự động tắt", + "Seconds used to evaluate timeout counts.": "Số giây dùng để đánh giá số lần chờ quá hạn.", + "Minimum timeout samples": "Số mẫu chờ quá hạn tối thiểu", + "Minimum observations before ratio-based disable is eligible.": "Số quan sát tối thiểu trước khi đủ điều kiện tắt theo tỷ lệ.", + "Timeout ratio threshold": "Ngưỡng tỷ lệ chờ quá hạn", + "Percentage threshold for the existing auto-disable policy.": "Ngưỡng phần trăm của chính sách tự động tắt hiện có.", + "Auto-disable duration": "Thời lượng tự động tắt", + "Seconds a channel remains disabled before recovery checks.": "Số giây kênh bị tắt trước khi kiểm tra khôi phục.", + "Successful recovery probes": "Lần thăm dò khôi phục thành công", + "Successful probes required before automatic recovery.": "Số lần thăm dò thành công cần có trước khi tự động khôi phục.", + "Expose channel demotion and disable transitions to the SmartOps alert stream.": "Đưa các chuyển đổi hạ ưu tiên và tắt kênh vào luồng cảnh báo SmartOps.", + "Save timeout policy": "Lưu chính sách thời gian chờ", + "Channel alert stream": "Luồng cảnh báo kênh", + "Live channel timeout transitions from the existing SmartOps alert endpoint.": "Các chuyển đổi thời gian chờ kênh trực tiếp từ API cảnh báo SmartOps hiện có.", + "No active channel timeout alerts.": "Không có cảnh báo thời gian chờ kênh đang hoạt động.", + "Unknown channel": "Kênh không xác định", + "Channel {{id}}": "Kênh {{id}}", + "This process is not currently reporting sustained host pressure, channel timeout, or billing reconciliation alerts.": "Tiến trình này hiện không báo cáo áp lực máy chủ kéo dài, thời gian chờ kênh hoặc cảnh báo đối soát thanh toán." } } From 029c5cd367297a9b5b6a546f3f670ddfeea5727b Mon Sep 17 00:00:00 2001 From: CSCITech Date: Sat, 3 Oct 2026 02:55:52 +0800 Subject: [PATCH 05/12] fix: close channel health review follow-ups --- pkg/channelhealth/runtime.go | 63 ++++++++++++------- pkg/channelhealth/runtime_test.go | 22 +++++++ service/smart_ops_alert.go | 8 +++ service/smart_ops_alert_delivery_test.go | 8 +++ setting/operation_setting/monitor_setting.go | 3 +- .../operation_setting/monitor_setting_test.go | 3 + .../smart-ops/channel-health/index.tsx | 2 +- web/default/src/i18n/locales/fr.json | 4 +- web/default/src/i18n/locales/ja.json | 2 +- web/default/src/i18n/locales/ru.json | 2 +- web/default/src/i18n/locales/vi.json | 2 +- web/default/src/i18n/locales/zh.json | 2 +- 12 files changed, 91 insertions(+), 30 deletions(-) diff --git a/pkg/channelhealth/runtime.go b/pkg/channelhealth/runtime.go index 4e08252e..26909040 100644 --- a/pkg/channelhealth/runtime.go +++ b/pkg/channelhealth/runtime.go @@ -26,6 +26,7 @@ const ( runtimeStateKeyPrefix = "maxapi:channel-timeout:v1:state:" runtimeStateIndexKey = "maxapi:channel-timeout:v1:index" runtimeStateIndexMigratedKey = "maxapi:channel-timeout:v1:index:migrated" + runtimeStateIndexCursorKey = "maxapi:channel-timeout:v1:index:migration-cursor" runtimeDedupeKeyPrefix = "maxapi:channel-timeout:v1:dedupe:" runtimeStateTTL = 7 * 24 * time.Hour runtimeDecisionTimeout = 100 * time.Millisecond @@ -273,24 +274,27 @@ func ListRuntimeStates(ctx context.Context) ([]RuntimeState, error) { if err != nil { return nil, err } - if len(members) > 0 { - return listRuntimeStatesByMembers(ctx, members) - } migrated, err := maxcommon.RDB.Exists(ctx, runtimeStateIndexMigratedKey).Result() if err != nil { return nil, err } if migrated > 0 { - return []RuntimeState{}, nil + return listRuntimeStatesByMembers(ctx, members) } // States written before the index was introduced are discovered by a bounded - // legacy scan and added to the index below. A complete scan records a marker; - // an interrupted scan remains eligible for a later migration attempt. New - // state transitions add their channel ID in the same Redis transaction, so - // normal requests never scan the shared keyspace after migration completes. + // legacy scan and added to the index below. The cursor is persisted so an + // interrupted scan resumes where it stopped; a complete scan records a + // marker. New state transitions add their channel ID in the same Redis + // transaction, so normal requests never scan the shared keyspace after + // migration completes. keys := make([]string, 0, 32) var cursor uint64 + if value, err := maxcommon.RDB.Get(ctx, runtimeStateIndexCursorKey).Result(); err != nil && !errors.Is(err, redis.Nil) { + return nil, err + } else if err == nil { + cursor, _ = strconv.ParseUint(value, 10, 64) + } scanComplete := false for { batch, next, err := maxcommon.RDB.Scan(ctx, cursor, runtimeStateKeyPrefix+"*", 200).Result() @@ -307,17 +311,22 @@ func ListRuntimeStates(ctx context.Context) ([]RuntimeState, error) { break } } - if len(keys) == 0 { - if scanComplete { - if err := maxcommon.RDB.Set(ctx, runtimeStateIndexMigratedKey, "1", 0).Err(); err != nil { - return nil, err - } + allMembers := make([]string, 0, len(members)+len(keys)) + seenMembers := make(map[string]struct{}, len(members)+len(keys)) + for _, member := range append(members, keys...) { + member = strings.TrimPrefix(member, runtimeStateKeyPrefix) + if _, ok := seenMembers[member]; ok { + continue } - return []RuntimeState{}, nil + seenMembers[member] = struct{}{} + allMembers = append(allMembers, member) } - members = make([]string, 0, len(keys)) - for _, key := range keys { - members = append(members, strings.TrimPrefix(key, runtimeStateKeyPrefix)) + members = allMembers + if len(members) == 0 && !scanComplete { + if err := maxcommon.RDB.Set(ctx, runtimeStateIndexCursorKey, strconv.FormatUint(cursor, 10), 0).Err(); err != nil { + return nil, err + } + return []RuntimeState{}, nil } states, err := listRuntimeStatesByMembers(ctx, members) if err != nil { @@ -327,13 +336,23 @@ func ListRuntimeStates(ctx context.Context) ([]RuntimeState, error) { for _, state := range states { validMembers = append(validMembers, strconv.Itoa(state.ChannelID)) } - if len(validMembers) > 0 { - if err := maxcommon.RDB.SAdd(ctx, runtimeStateIndexKey, validMembers).Err(); err != nil { + if scanComplete { + pipe := maxcommon.RDB.TxPipeline() + if len(validMembers) > 0 { + pipe.SAdd(ctx, runtimeStateIndexKey, validMembers) + } + pipe.Set(ctx, runtimeStateIndexMigratedKey, "1", 0) + pipe.Del(ctx, runtimeStateIndexCursorKey) + if _, err := pipe.Exec(ctx); err != nil { return nil, err } - } - if scanComplete { - if err := maxcommon.RDB.Set(ctx, runtimeStateIndexMigratedKey, "1", 0).Err(); err != nil { + } else { + pipe := maxcommon.RDB.TxPipeline() + if len(validMembers) > 0 { + pipe.SAdd(ctx, runtimeStateIndexKey, validMembers) + } + pipe.Set(ctx, runtimeStateIndexCursorKey, strconv.FormatUint(cursor, 10), 0) + if _, err := pipe.Exec(ctx); err != nil { return nil, err } } diff --git a/pkg/channelhealth/runtime_test.go b/pkg/channelhealth/runtime_test.go index 4a088b8f..b89fcfe7 100644 --- a/pkg/channelhealth/runtime_test.go +++ b/pkg/channelhealth/runtime_test.go @@ -432,6 +432,28 @@ func TestListRuntimeStatesMarksLegacyScanComplete(t *testing.T) { require.Empty(t, states) } +func TestListRuntimeStatesCompletesLegacyMigrationWithIndexedMembers(t *testing.T) { + withRuntimeRedis(t) + for channelID, penalty := range map[int]string{937: "5", 938: "7"} { + _, err := maxcommon.RDB.HSet(context.Background(), RuntimeStateKey(channelID), map[string]interface{}{ + "channel_id": strconv.Itoa(channelID), + "penalty": penalty, + }).Result() + require.NoError(t, err) + } + require.NoError(t, maxcommon.RDB.SAdd(context.Background(), runtimeStateIndexKey, "937").Err()) + + states, err := ListRuntimeStates(context.Background()) + require.NoError(t, err) + require.Len(t, states, 2) + migrated, err := maxcommon.RDB.Exists(context.Background(), runtimeStateIndexMigratedKey).Result() + require.NoError(t, err) + require.Equal(t, int64(1), migrated) + indexed, err := maxcommon.RDB.SIsMember(context.Background(), runtimeStateIndexKey, "938").Result() + require.NoError(t, err) + require.True(t, indexed) +} + func TestRecoverRuntimeStateRequiresConfiguredSuccessfulProbes(t *testing.T) { withRuntimeRedis(t) setting := testRuntimeSetting() diff --git a/service/smart_ops_alert.go b/service/smart_ops_alert.go index bda0770e..67e0583f 100644 --- a/service/smart_ops_alert.go +++ b/service/smart_ops_alert.go @@ -1322,6 +1322,7 @@ var smartOpsAlertSendToRecipient = func(recipient smartOpsAlertRecipient, data d } func enqueueSmartOpsAlertNotification(alert SmartOpsAlert) { + alert = normalizeSmartOpsAlertObservedAt(alert) smartOpsAlertNotificationQueue.Do(func() { smartOpsAlertNotificationQueue.pool = newSmartOpsAlertNotificationPool( smartOpsAlertWorkerCount, @@ -1350,6 +1351,13 @@ func enqueueSmartOpsAlertNotification(alert SmartOpsAlert) { } } +func normalizeSmartOpsAlertObservedAt(alert SmartOpsAlert) SmartOpsAlert { + if alert.ObservedAt.IsZero() { + alert.ObservedAt = time.Now() + } + return alert +} + func smartOpsAlertNotification(alert SmartOpsAlert) dto.Notify { statusText := "异常" if alert.Status == smartOpsAlertStatusResolved { diff --git a/service/smart_ops_alert_delivery_test.go b/service/smart_ops_alert_delivery_test.go index c4fb27a8..760a1c88 100644 --- a/service/smart_ops_alert_delivery_test.go +++ b/service/smart_ops_alert_delivery_test.go @@ -65,6 +65,14 @@ func TestSmartOpsAlertDeliveryProjectionRejectsOlderStatus(t *testing.T) { require.Equal(t, 1, state.DeliveryAttempts) } +func TestNormalizeSmartOpsAlertObservedAtKeepsOneDeliveryTimestamp(t *testing.T) { + alert := normalizeSmartOpsAlertObservedAt(SmartOpsAlert{Key: "channel_timeout:936"}) + require.False(t, alert.ObservedAt.IsZero()) + + normalizedAgain := normalizeSmartOpsAlertObservedAt(alert) + require.Equal(t, alert.ObservedAt, normalizedAgain.ObservedAt) +} + func TestDeliverSmartOpsAlertRecordsSentDeliveryState(t *testing.T) { withSmartOpsDeliveryRedis(t) originalDelays := smartOpsAlertRetryDelays diff --git a/setting/operation_setting/monitor_setting.go b/setting/operation_setting/monitor_setting.go index 296e4596..768edf0c 100644 --- a/setting/operation_setting/monitor_setting.go +++ b/setting/operation_setting/monitor_setting.go @@ -2,6 +2,7 @@ package operation_setting import ( "fmt" + "math" "os" "strconv" "strings" @@ -162,7 +163,7 @@ func ValidateMonitorSettingOption(key, value string) error { return validateMonitorInt(key, value, 0, 2592000) case "timeout_auto_disable_ratio_percent": ratio, err := strconv.ParseFloat(strings.TrimSpace(value), 64) - if err != nil || ratio < 0 || ratio > 100 { + if err != nil || math.IsNaN(ratio) || ratio < 0 || ratio > 100 { return fmt.Errorf("%s must be between 0 and 100", key) } case "timeout_auto_disable_count_scope": diff --git a/setting/operation_setting/monitor_setting_test.go b/setting/operation_setting/monitor_setting_test.go index 9eb7baff..90cd194d 100644 --- a/setting/operation_setting/monitor_setting_test.go +++ b/setting/operation_setting/monitor_setting_test.go @@ -34,6 +34,9 @@ func TestValidateMonitorSettingOption(t *testing.T) { require.Error(t, ValidateMonitorSettingOption( "monitor_setting.priority_deduction", "-1", )) + require.Error(t, ValidateMonitorSettingOption( + "monitor_setting.timeout_auto_disable_ratio_percent", "NaN", + )) } func TestMonitorSettingNormalizationRunsOnConfigLoad(t *testing.T) { diff --git a/web/default/src/features/smart-ops/channel-health/index.tsx b/web/default/src/features/smart-ops/channel-health/index.tsx index 53f5b2bb..445d214d 100644 --- a/web/default/src/features/smart-ops/channel-health/index.tsx +++ b/web/default/src/features/smart-ops/channel-health/index.tsx @@ -471,7 +471,7 @@ export function ChannelHealth(): ReactElement { if (updates.length === 0) { return } - await updatePolicy.mutateAsync(updates) + updatePolicy.mutate(updates) } const setNumber = (name: keyof ChannelHealthDraft, value: number) => diff --git a/web/default/src/i18n/locales/fr.json b/web/default/src/i18n/locales/fr.json index b8045e40..24929976 100644 --- a/web/default/src/i18n/locales/fr.json +++ b/web/default/src/i18n/locales/fr.json @@ -5764,7 +5764,7 @@ "Delivery status": "État de livraison", "Recovery mode": "Mode de récupération", "Automatic": "Automatique", - "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "La récupération automatique rétablit la pénalité après son délai de refroidissement ; la récupération manuelle la conserve jusqu’à l’intervention d’un administrateur.", + "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "La récupération automatique lève la pénalité après son délai de refroidissement ; la récupération manuelle la conserve jusqu’à l’intervention d’un administrateur.", "Extend penalty on repeated timeout": "Prolonger la pénalité en cas de dépassement répété", "Keep the current penalty deadline when repeated timeouts continue during an active penalty.": "Conserver l’échéance actuelle de la pénalité lorsque les dépassements répétés se poursuivent pendant une pénalité active.", "{{count}} priority point": "{{count}} point de priorité", @@ -5776,7 +5776,7 @@ "Priority deduction must be a non-negative integer.": "La déduction de priorité doit être un entier non négatif.", "Disable count must be at least 2.": "Le nombre de désactivations doit être au moins égal à 2.", "Disable window must be between 60 and 2592000 seconds.": "La fenêtre de désactivation doit être comprise entre 60 et 2592000 secondes.", - "Minimum samples must be a non-negative integer.": "Minimum samples must be a non-negative integer.", + "Minimum samples must be a non-negative integer.": "Le nombre minimal d’échantillons doit être un entier non négatif.", "Timeout ratio must be between 0 and 100 percent.": "Le ratio de dépassement doit être compris entre 0 et 100 %.", "Disable duration must be between 0 and 604800 seconds.": "La durée de désactivation doit être comprise entre 0 et 604800 secondes.", "Successful probe count must be between 1 and 10.": "Le nombre de sondes réussies doit être compris entre 1 et 10.", diff --git a/web/default/src/i18n/locales/ja.json b/web/default/src/i18n/locales/ja.json index 56df2467..a9cfc308 100644 --- a/web/default/src/i18n/locales/ja.json +++ b/web/default/src/i18n/locales/ja.json @@ -5776,7 +5776,7 @@ "Priority deduction must be a non-negative integer.": "優先度の減点は 0 以上の整数で指定してください。", "Disable count must be at least 2.": "無効化回数は 2 以上で指定してください。", "Disable window must be between 60 and 2592000 seconds.": "無効化ウィンドウは 60～2592000 秒で指定してください。", - "Minimum samples must be a non-negative integer.": "Minimum samples must be a non-negative integer.", + "Minimum samples must be a non-negative integer.": "最小サンプル数は 0 以上の整数で指定してください。", "Timeout ratio must be between 0 and 100 percent.": "タイムアウト比率は 0～100 パーセントで指定してください。", "Disable duration must be between 0 and 604800 seconds.": "無効化期間は 0～604800 秒で指定してください。", "Successful probe count must be between 1 and 10.": "成功プローブ数は 1～10 で指定してください。", diff --git a/web/default/src/i18n/locales/ru.json b/web/default/src/i18n/locales/ru.json index 6e3f7613..c1ebaec9 100644 --- a/web/default/src/i18n/locales/ru.json +++ b/web/default/src/i18n/locales/ru.json @@ -5784,7 +5784,7 @@ "Priority deduction must be a non-negative integer.": "Снижение приоритета должно быть неотрицательным целым числом.", "Disable count must be at least 2.": "Число отключений должно быть не меньше 2.", "Disable window must be between 60 and 2592000 seconds.": "Окно отключения должно быть от 60 до 2592000 секунд.", - "Minimum samples must be a non-negative integer.": "Minimum samples must be a non-negative integer.", + "Minimum samples must be a non-negative integer.": "Минимальное число образцов должно быть неотрицательным целым числом.", "Timeout ratio must be between 0 and 100 percent.": "Доля таймаутов должна быть от 0 до 100 процентов.", "Disable duration must be between 0 and 604800 seconds.": "Длительность отключения должна быть от 0 до 604800 секунд.", "Successful probe count must be between 1 and 10.": "Число успешных проверок должно быть от 1 до 10.", diff --git a/web/default/src/i18n/locales/vi.json b/web/default/src/i18n/locales/vi.json index 58017fa1..a7a21eaf 100644 --- a/web/default/src/i18n/locales/vi.json +++ b/web/default/src/i18n/locales/vi.json @@ -5776,7 +5776,7 @@ "Priority deduction must be a non-negative integer.": "Mức trừ ưu tiên phải là số nguyên không âm.", "Disable count must be at least 2.": "Số lần tắt phải ít nhất là 2.", "Disable window must be between 60 and 2592000 seconds.": "Cửa sổ tắt phải từ 60 đến 2592000 giây.", - "Minimum samples must be a non-negative integer.": "Minimum samples must be a non-negative integer.", + "Minimum samples must be a non-negative integer.": "Số mẫu tối thiểu phải là số nguyên không âm.", "Timeout ratio must be between 0 and 100 percent.": "Tỷ lệ chờ quá hạn phải từ 0 đến 100 phần trăm.", "Disable duration must be between 0 and 604800 seconds.": "Thời lượng tắt phải từ 0 đến 604800 giây.", "Successful probe count must be between 1 and 10.": "Số lần thăm dò thành công phải từ 1 đến 10.", diff --git a/web/default/src/i18n/locales/zh.json b/web/default/src/i18n/locales/zh.json index bebaaa61..c5a32200 100644 --- a/web/default/src/i18n/locales/zh.json +++ b/web/default/src/i18n/locales/zh.json @@ -5624,7 +5624,7 @@ "Delivery status": "投递状态", "Recovery mode": "恢复模式", "Automatic": "自动", - "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "自动恢复会在冷却时间后恢复惩罚；手动恢复会保留惩罚，直到管理员操作。", + "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "自动恢复会在冷却时间后解除惩罚；手动恢复会保留惩罚，直到管理员操作。", "Extend penalty on repeated timeout": "重复超时时延长惩罚", "Keep the current penalty deadline when repeated timeouts continue during an active penalty.": "惩罚生效期间再次超时时，是否保留并延长当前惩罚截止时间。", "Expose channel demotion and disable transitions to the SmartOps alert stream.": "将渠道降级和禁用状态变化发布到 SmartOps 告警流。", From df192ffb82446cc200b02263ff9ed3d79dff7166 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Sat, 3 Oct 2026 03:53:07 +0800 Subject: [PATCH 06/12] fix: exclude channel tests from health penalties --- .../common/channel_health_observation_test.go | 49 +++++++++++++++++++ relay/common/relay_info.go | 9 ++-- setting/operation_setting/monitor_setting.go | 4 +- .../operation_setting/monitor_setting_test.go | 3 ++ 4 files changed, 60 insertions(+), 5 deletions(-) diff --git a/relay/common/channel_health_observation_test.go b/relay/common/channel_health_observation_test.go index 2d0395e8..1967946e 100644 --- a/relay/common/channel_health_observation_test.go +++ b/relay/common/channel_health_observation_test.go @@ -95,3 +95,52 @@ func TestRelayInfoTransportTimeoutHonorsConfiguredThreshold(t *testing.T) { require.Zero(t, state.Penalty) require.Zero(t, state.TimeoutCount[channelhealth.RequestModeStreaming]) } + +func TestRelayInfoChannelTestDoesNotRecordHealthEvidence(t *testing.T) { + server := miniredis.RunT(t) + oldRDB, oldEnabled := maxcommon.RDB, maxcommon.RedisEnabled + maxcommon.RDB = redis.NewClient(&redis.Options{Addr: server.Addr()}) + maxcommon.RedisEnabled = true + t.Cleanup(func() { + _ = maxcommon.RDB.Close() + maxcommon.RDB, maxcommon.RedisEnabled = oldRDB, oldEnabled + }) + + oldRecorder := recordChannelHealthObservation + var observations []maxcommon.ChannelHealthObservation + recordChannelHealthObservation = func(observation maxcommon.ChannelHealthObservation) error { + observations = append(observations, observation) + return nil + } + t.Cleanup(func() { recordChannelHealthObservation = oldRecorder }) + + oldSetting := *operation_setting.GetMonitorSetting() + *operation_setting.GetMonitorSetting() = operation_setting.MonitorSetting{ + AutoPriorityDemotionEnabled: true, + TimeoutAutoDisableEnabled: true, + TimeoutAutoDisableMinimumSamples: 1, + StreamingFirstResultTimeoutSeconds: 1, + PriorityDeduction: 10, + PenaltyCooldownSeconds: 60, + } + t.Cleanup(func() { *operation_setting.GetMonitorSetting() = oldSetting }) + + start := time.Now().Add(-2 * time.Second) + info := &RelayInfo{ + StartTime: start, + IsStream: true, + IsChannelTest: true, + isFirstResponse: true, + ChannelMeta: &ChannelMeta{ChannelId: 936, ChannelAutoBan: true}, + channelAttemptStartTime: start, + } + info.SetFirstResponseTime() + info.RecordChannelTimeout("transport_timeout") + time.Sleep(50 * time.Millisecond) + + require.Empty(t, observations) + state, err := channelhealth.LoadRuntimeState(context.Background(), 936) + require.NoError(t, err) + require.Zero(t, state.Penalty) + require.Zero(t, state.TimeoutCount[channelhealth.RequestModeStreaming]) +} diff --git a/relay/common/relay_info.go b/relay/common/relay_info.go index a70e4e8b..bb4e6fc3 100644 --- a/relay/common/relay_info.go +++ b/relay/common/relay_info.go @@ -781,6 +781,9 @@ func (info *RelayInfo) setFirstResponseTimeLocked() { if info.channelFirstResponseSignal != nil { close(info.channelFirstResponseSignal) } + if info.IsChannelTest { + return + } info.recordChannelHealthObservation("first_response") info.recordChannelHealthSuccess() } @@ -886,7 +889,7 @@ func (info *RelayInfo) channelHealthAttemptID() string { // when the administrator enabled the advanced timeout gates. It runs off the // relay response path so a slow Redis instance cannot delay the client. func (info *RelayInfo) recordChannelHealthSuccess() { - if info == nil || info.ChannelMeta == nil || info.ChannelId <= 0 || !info.ChannelAutoBan { + if info == nil || info.IsChannelTest || info.ChannelMeta == nil || info.ChannelId <= 0 || !info.ChannelAutoBan { return } setting := operation_setting.GetMonitorSetting() @@ -917,10 +920,10 @@ func (info *RelayInfo) recordChannelHealthSuccess() { // RecordChannelTimeout records a confirmed timeout outcome for the current // channel attempt. It is intentionally side-effect-free for relay behavior. func (info *RelayInfo) RecordChannelTimeout(event string) { - info.recordChannelHealthObservation(event) - if info == nil { + if info == nil || info.IsChannelTest { return } + info.recordChannelHealthObservation(event) timeoutKind := event observedAt := time.Now() if event == "transport_timeout" || event == "request_timeout" { diff --git a/setting/operation_setting/monitor_setting.go b/setting/operation_setting/monitor_setting.go index 768edf0c..7dbc237a 100644 --- a/setting/operation_setting/monitor_setting.go +++ b/setting/operation_setting/monitor_setting.go @@ -162,7 +162,7 @@ func ValidateMonitorSettingOption(key, value string) error { case "penalty_cooldown_seconds", "alert_repeat_interval_seconds": return validateMonitorInt(key, value, 0, 2592000) case "timeout_auto_disable_ratio_percent": - ratio, err := strconv.ParseFloat(strings.TrimSpace(value), 64) + ratio, err := strconv.ParseFloat(value, 64) if err != nil || math.IsNaN(ratio) || ratio < 0 || ratio > 100 { return fmt.Errorf("%s must be between 0 and 100", key) } @@ -183,7 +183,7 @@ func ValidateMonitorSettingOption(key, value string) error { } func validateMonitorInt(key, value string, lo, hi int) error { - parsed, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64) + parsed, err := strconv.ParseInt(value, 10, 64) if err != nil || parsed < int64(lo) || parsed > int64(hi) { return fmt.Errorf("%s must be an integer between %d and %d", key, lo, hi) } diff --git a/setting/operation_setting/monitor_setting_test.go b/setting/operation_setting/monitor_setting_test.go index 90cd194d..5da655b3 100644 --- a/setting/operation_setting/monitor_setting_test.go +++ b/setting/operation_setting/monitor_setting_test.go @@ -37,6 +37,9 @@ func TestValidateMonitorSettingOption(t *testing.T) { require.Error(t, ValidateMonitorSettingOption( "monitor_setting.timeout_auto_disable_ratio_percent", "NaN", )) + require.Error(t, ValidateMonitorSettingOption( + "monitor_setting.streaming_first_result_timeout_seconds", " 15", + )) } func TestMonitorSettingNormalizationRunsOnConfigLoad(t *testing.T) { From 0bfd0b5b5aea3e5355b8831138104bcd606580a2 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Sat, 3 Oct 2026 04:38:45 +0800 Subject: [PATCH 07/12] fix: close final channel health review findings --- relay/channel/api_request.go | 41 +++++++++++++++++++ relay/channel/api_request_test.go | 28 +++++++++++++ relay/common/relay_info.go | 33 ++++++++++++++- .../stream_first_result_timeout_test.go | 9 ++++ relay/helper/stream_scanner.go | 7 +++- service/smart_ops_alert.go | 4 +- service/smart_ops_alert_delivery_test.go | 6 +++ web/default/src/i18n/locales/zh.json | 2 +- 8 files changed, 126 insertions(+), 4 deletions(-) diff --git a/relay/channel/api_request.go b/relay/channel/api_request.go index 4fe90f9c..3e4e9ee4 100644 --- a/relay/channel/api_request.go +++ b/relay/channel/api_request.go @@ -518,6 +518,7 @@ func doRequest(c *gin.Context, req *http.Request, info *common.RelayInfo) (*http client = service.GetHttpClient() } client = clientWithNonStreamingTimeout(client, info) + client = clientWithStreamingHeaderTimeout(client, info) var stopPinger context.CancelFunc var pingerDone <-chan struct{} @@ -583,6 +584,46 @@ func clientWithNonStreamingTimeout(client *http.Client, info *common.RelayInfo) return &clone } +// clientWithStreamingHeaderTimeout bounds the wait for upstream response +// headers by the same first-response budget used by StreamScannerHandler. +// The timeout is applied to a cloned transport, so it ends when headers are +// received and does not cap the rest of a successful stream. +func clientWithStreamingHeaderTimeout(client *http.Client, info *common.RelayInfo) *http.Client { + if client == nil || info == nil || !info.IsStream { + return client + } + setting := operation_setting.GetMonitorSetting() + if setting == nil || setting.StreamingFirstResultTimeoutSeconds <= 0 { + return client + } + remaining := common.RemainingChannelFirstResponseTimeout( + info.ChannelAttemptStartTime(), + time.Duration(setting.StreamingFirstResultTimeoutSeconds)*time.Second, + time.Now(), + ) + if remaining <= 0 { + return client + } + + transport, ok := client.Transport.(*http.Transport) + if client.Transport == nil { + transport, ok = http.DefaultTransport.(*http.Transport) + } + if !ok || transport == nil { + // All built-in relay clients use *http.Transport. Custom round + // trippers are left untouched because imposing http.Client.Timeout + // would also terminate the body after a successful first response. + return client + } + clonedTransport := transport.Clone() + if clonedTransport.ResponseHeaderTimeout == 0 || remaining < clonedTransport.ResponseHeaderTimeout { + clonedTransport.ResponseHeaderTimeout = remaining + } + clone := *client + clone.Transport = clonedTransport + return &clone +} + type timeoutTrackingBody struct { io.ReadCloser info *common.RelayInfo diff --git a/relay/channel/api_request_test.go b/relay/channel/api_request_test.go index 06f4ec69..ca5a48ac 100644 --- a/relay/channel/api_request_test.go +++ b/relay/channel/api_request_test.go @@ -16,10 +16,38 @@ import ( "github.com/MAX-API-Next/MAX-API/relay/channel/task/taskcommon" relaycommon "github.com/MAX-API-Next/MAX-API/relay/common" "github.com/MAX-API-Next/MAX-API/service" + "github.com/MAX-API-Next/MAX-API/setting/operation_setting" "github.com/gin-gonic/gin" "github.com/stretchr/testify/require" ) +func TestClientWithStreamingHeaderTimeoutUsesRemainingAttemptBudget(t *testing.T) { + originalSetting := *operation_setting.GetMonitorSetting() + *operation_setting.GetMonitorSetting() = operation_setting.MonitorSetting{ + StreamingFirstResultTimeoutSeconds: 1, + } + t.Cleanup(func() { *operation_setting.GetMonitorSetting() = originalSetting }) + + transport := &http.Transport{ResponseHeaderTimeout: 5 * time.Second} + client := &http.Client{Transport: transport} + info := &relaycommon.RelayInfo{ + IsStream: true, + } + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + ctx.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil) + info.InitChannelMeta(ctx) + time.Sleep(400 * time.Millisecond) + + cloned := clientWithStreamingHeaderTimeout(client, info) + require.NotSame(t, client, cloned) + clonedTransport, ok := cloned.Transport.(*http.Transport) + require.True(t, ok) + require.Greater(t, clonedTransport.ResponseHeaderTimeout, 500*time.Millisecond) + require.Less(t, clonedTransport.ResponseHeaderTimeout, 700*time.Millisecond) + require.Equal(t, 5*time.Second, transport.ResponseHeaderTimeout) +} + type taskHeaderAdaptor struct { taskcommon.BaseBilling url string diff --git a/relay/common/relay_info.go b/relay/common/relay_info.go index bb4e6fc3..7b98d9e3 100644 --- a/relay/common/relay_info.go +++ b/relay/common/relay_info.go @@ -838,6 +838,36 @@ func (info *RelayInfo) FirstResponseSignal() <-chan struct{} { return info.channelFirstResponseSignal } +// ChannelAttemptStartTime returns the start of the current channel attempt. +// The value is copied so callers can calculate a deadline without exposing +// RelayInfo's mutable retry state. +func (info *RelayInfo) ChannelAttemptStartTime() time.Time { + if info == nil { + return time.Time{} + } + return info.channelAttemptStartTime +} + +// RemainingChannelFirstResponseTimeout returns the portion of a configured +// first-response timeout that remains for the current channel attempt. A zero +// start time means the attempt start is unavailable, so the full configured +// timeout is retained. Once the budget is exhausted, a nanosecond is returned +// so time.NewTimer still fires immediately instead of treating zero as +// "disabled". +func RemainingChannelFirstResponseTimeout(start time.Time, configured time.Duration, now time.Time) time.Duration { + if configured <= 0 { + return 0 + } + if start.IsZero() { + return configured + } + remaining := configured - now.Sub(start) + if remaining <= 0 { + return time.Nanosecond + } + return remaining +} + func (info *RelayInfo) HasRecordedChannelFirstResponse() bool { return info != nil && info.channelFirstResponseRecorded.Load() } @@ -897,6 +927,7 @@ func (info *RelayInfo) recordChannelHealthSuccess() { (setting.TimeoutAutoDisableMinimumSamples <= 0 && setting.TimeoutAutoDisableRatioPercent <= 0) { return } + settingSnapshot := *setting requestMode := channelhealth.RequestModeNonStreaming if info.IsStream { requestMode = channelhealth.RequestModeStreaming @@ -913,7 +944,7 @@ func (info *RelayInfo) recordChannelHealthSuccess() { go func() { decisionContext, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() - _, _ = channelhealth.EvaluateTimeoutGuard(decisionContext, evidence, setting) + _, _ = channelhealth.EvaluateTimeoutGuard(decisionContext, evidence, &settingSnapshot) }() } diff --git a/relay/helper/stream_first_result_timeout_test.go b/relay/helper/stream_first_result_timeout_test.go index f812b92d..d0d4e8b8 100644 --- a/relay/helper/stream_first_result_timeout_test.go +++ b/relay/helper/stream_first_result_timeout_test.go @@ -71,3 +71,12 @@ func TestStreamScannerHandlerDoesNotTreatUsageOnlyFrameAsFirstResult(t *testing. require.True(t, info.HasRecordedChannelFirstResponse()) require.NotEqual(t, relaycommon.StreamEndReasonTimeout, info.StreamStatus.EndReason) } + +func TestRemainingFirstResultTimeoutUsesChannelAttemptBudget(t *testing.T) { + configured := time.Second + start := time.Now().Add(-400 * time.Millisecond) + remaining := relaycommon.RemainingChannelFirstResponseTimeout(start, configured, time.Now()) + + require.Greater(t, remaining, 500*time.Millisecond) + require.Less(t, remaining, 700*time.Millisecond) +} diff --git a/relay/helper/stream_scanner.go b/relay/helper/stream_scanner.go index 8c8b66be..8c432e22 100644 --- a/relay/helper/stream_scanner.go +++ b/relay/helper/stream_scanner.go @@ -96,7 +96,12 @@ func StreamScannerHandler(c *gin.Context, resp *http.Response, info *relaycommon var firstResultTimer *time.Timer var firstResultTimeoutChan <-chan time.Time if firstResultTimeoutSeconds > 0 && info.FirstResponseSignal() != nil { - firstResultTimer = time.NewTimer(time.Duration(firstResultTimeoutSeconds) * time.Second) + firstResultTimeout := relaycommon.RemainingChannelFirstResponseTimeout( + info.ChannelAttemptStartTime(), + time.Duration(firstResultTimeoutSeconds)*time.Second, + time.Now(), + ) + firstResultTimer = time.NewTimer(firstResultTimeout) firstResultTimeoutChan = firstResultTimer.C } diff --git a/service/smart_ops_alert.go b/service/smart_ops_alert.go index 67e0583f..c218af05 100644 --- a/service/smart_ops_alert.go +++ b/service/smart_ops_alert.go @@ -1430,8 +1430,10 @@ func smartOpsAlertDeliveryStatusRank(status string) int { return 40 case "skipped_repeat": return 50 - default: + case "failed", "delivery_unknown": return 30 + default: + return 0 } } diff --git a/service/smart_ops_alert_delivery_test.go b/service/smart_ops_alert_delivery_test.go index 760a1c88..0a12fc48 100644 --- a/service/smart_ops_alert_delivery_test.go +++ b/service/smart_ops_alert_delivery_test.go @@ -65,6 +65,12 @@ func TestSmartOpsAlertDeliveryProjectionRejectsOlderStatus(t *testing.T) { require.Equal(t, 1, state.DeliveryAttempts) } +func TestSmartOpsAlertDeliveryStatusRankRejectsUnknownStatuses(t *testing.T) { + require.Equal(t, 30, smartOpsAlertDeliveryStatusRank("failed")) + require.Equal(t, 30, smartOpsAlertDeliveryStatusRank("delivery_unknown")) + require.Zero(t, smartOpsAlertDeliveryStatusRank("unexpected_status")) +} + func TestNormalizeSmartOpsAlertObservedAtKeepsOneDeliveryTimestamp(t *testing.T) { alert := normalizeSmartOpsAlertObservedAt(SmartOpsAlert{Key: "channel_timeout:936"}) require.False(t, alert.ObservedAt.IsZero()) diff --git a/web/default/src/i18n/locales/zh.json b/web/default/src/i18n/locales/zh.json index c5a32200..b81b6238 100644 --- a/web/default/src/i18n/locales/zh.json +++ b/web/default/src/i18n/locales/zh.json @@ -5626,7 +5626,7 @@ "Automatic": "自动", "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "自动恢复会在冷却时间后解除惩罚；手动恢复会保留惩罚，直到管理员操作。", "Extend penalty on repeated timeout": "重复超时时延长惩罚", - "Keep the current penalty deadline when repeated timeouts continue during an active penalty.": "惩罚生效期间再次超时时，是否保留并延长当前惩罚截止时间。", + "Keep the current penalty deadline when repeated timeouts continue during an active penalty.": "惩罚生效期间再次超时时，保留当前惩罚截止时间。", "Expose channel demotion and disable transitions to the SmartOps alert stream.": "将渠道降级和禁用状态变化发布到 SmartOps 告警流。", "Save timeout policy": "保存超时策略", "Channel alert stream": "渠道告警流", From a688f1657cdab8d62633cc53f84128f1cf1348d7 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Sat, 3 Oct 2026 05:16:20 +0800 Subject: [PATCH 08/12] fix: harden channel health runtime projections --- pkg/channelhealth/runtime.go | 42 ++++++++++++--- pkg/channelhealth/runtime_test.go | 54 +++++++++++++++++++ relay/channel/api_request.go | 33 +++++++----- relay/channel/api_request_test.go | 15 +++--- .../common/channel_health_observation_test.go | 48 +++++++++++++++++ relay/common/relay_info.go | 18 +++++-- service/smart_ops_alert.go | 1 + service/smart_ops_alert_test.go | 30 +++++++++++ 8 files changed, 212 insertions(+), 29 deletions(-) diff --git a/pkg/channelhealth/runtime.go b/pkg/channelhealth/runtime.go index 26909040..42ab5613 100644 --- a/pkg/channelhealth/runtime.go +++ b/pkg/channelhealth/runtime.go @@ -33,6 +33,8 @@ const ( runtimeWatchAttempts = 32 ) +const removeStaleRuntimeMemberScript = `if redis.call("EXISTS", KEYS[1]) == 0 then return redis.call("SREM", KEYS[2], ARGV[1]) end return 0` + var ErrRuntimeUnavailable = errors.New("channel timeout runtime is unavailable") type AttemptEvidence struct { @@ -63,6 +65,7 @@ type RuntimeState struct { LastTimeoutMode string LastObservedAt time.Time RecoveryProbeSuccessCount int64 + RecoveryWatermark time.Time } type EvaluationResult struct { @@ -150,6 +153,13 @@ func decodeRuntimeState(channelID int, fields map[string]string) RuntimeState { state.LastTimeoutMode = normalizeRuntimeTimeoutMode(fields["last_timeout_mode"]) state.LastObservedAt = parseTime(fields["last_observed_at"]) state.RecoveryProbeSuccessCount = parseInt64(fields["recovery_probe_success_count"]) + state.RecoveryWatermark = parseTime(fields["recovery_watermark"]) + if state.RecoveryWatermark.IsZero() && isRuntimeRecoveryEvent(state.LastEvent) { + // Backfill the durable boundary for states written before the watermark + // field existed; the last recovery observation is the safest available + // lower bound. + state.RecoveryWatermark = state.LastObservedAt + } for _, mode := range []string{RequestModeStreaming, RequestModeNonStreaming, "combined"} { state.TimeoutCount[mode] = parseInt64(fields["timeout_count_"+mode]) state.SampleCount[mode] = parseInt64(fields["sample_count_"+mode]) @@ -171,6 +181,7 @@ func encodeRuntimeState(state RuntimeState) map[string]interface{} { "last_timeout_mode": state.LastTimeoutMode, "last_observed_at": formatTime(state.LastObservedAt), "recovery_probe_success_count": strconv.FormatInt(state.RecoveryProbeSuccessCount, 10), + "recovery_watermark": formatTime(state.RecoveryWatermark), } for _, mode := range []string{RequestModeStreaming, RequestModeNonStreaming, "combined"} { fields["timeout_count_"+mode] = strconv.FormatInt(state.TimeoutCount[mode], 10) @@ -382,7 +393,8 @@ func listRuntimeStatesByMembers(ctx context.Context, members []string) ([]Runtim return nil, err } if len(fields) == 0 { - _ = maxcommon.RDB.SRem(ctx, runtimeStateIndexKey, validMembers[index]).Err() + _ = maxcommon.RDB.Eval(ctx, removeStaleRuntimeMemberScript, + []string{runtimeStateKey(channelID), runtimeStateIndexKey}, validMembers[index]).Err() continue } states = append(states, decodeRuntimeState(channelID, fields)) @@ -436,9 +448,10 @@ func EvaluateTimeoutGuard(ctx context.Context, evidence AttemptEvidence, setting } state := decodeRuntimeState(evidence.ChannelID, fields) result.State = state - if isRuntimeRecoveryEvent(state.LastEvent) && !state.LastObservedAt.IsZero() && now.Before(state.LastObservedAt) { + if !state.RecoveryWatermark.IsZero() && now.Before(state.RecoveryWatermark) { // A timeout observed before the last recovery must not be allowed to - // reapply a penalty after the recovery transaction wins the watch. + // reapply a penalty after the recovery transaction wins the watch, + // even when a newer success sample has replaced LastEvent. return nil } if deduped > 0 { @@ -528,7 +541,7 @@ func EvaluateTimeoutGuard(ctx context.Context, evidence AttemptEvidence, setting state.Penalty > 0 && setting.RecoveryMode == operation_setting.RecoveryModeManual { pipe.Persist(ctx, key) } else { - pipe.Expire(ctx, key, runtimeStateTTL) + pipe.Expire(ctx, key, runtimeStateTTLFor(state, now)) } _, err = pipe.Exec(ctx) return err @@ -549,6 +562,21 @@ func boundedRuntimeContext(ctx context.Context) (context.Context, context.Cancel return context.WithTimeout(ctx, runtimeDecisionTimeout) } +func runtimeStateTTLFor(state RuntimeState, now time.Time) time.Duration { + ttl := runtimeStateTTL + for _, until := range []time.Time{state.PenaltyUntil, state.DisabledUntil} { + if until.IsZero() || !until.After(now) { + continue + } + // Keep a small grace period so Redis' whole-second TTL precision cannot + // remove an active penalty or disablement at its exact deadline. + if remaining := until.Sub(now) + time.Hour; remaining > ttl { + ttl = remaining + } + } + return ttl +} + // watchRuntimeState retries optimistic Redis transactions that lost a // concurrent update. The retry budget is bounded by the caller's decision // context so contention never turns into an unbounded relay delay. @@ -628,6 +656,7 @@ func RecoverRuntimeState(ctx context.Context, channelID int, probeID string, suc state.PenaltyUntil = time.Time{} state.RecoveryProbeSuccessCount = 0 state.LastEvent = "recovery_probe_succeeded" + state.RecoveryWatermark = now result.Transition = "runtime_recovered" result.Applied = true } else { @@ -644,7 +673,7 @@ func RecoverRuntimeState(ctx context.Context, channelID int, probeID string, suc pipe.HSet(decisionCtx, key, encodeRuntimeState(state)) pipe.SAdd(decisionCtx, runtimeStateIndexKey, strconv.Itoa(channelID)) pipe.Set(decisionCtx, dedupeKey, "1", probeTTL) - pipe.Expire(decisionCtx, key, runtimeStateTTL) + pipe.Expire(decisionCtx, key, runtimeStateTTLFor(state, now)) _, err = pipe.Exec(decisionCtx) return err }) @@ -688,13 +717,14 @@ func RestoreRuntimeState(ctx context.Context, channelID int) (EvaluationResult, state.Generation++ state.LastEvent = "manual_recovery" state.LastObservedAt = time.Now() + state.RecoveryWatermark = state.LastObservedAt result.State = state result.Transition = "runtime_recovered" result.Applied = true pipe := tx.TxPipeline() pipe.HSet(decisionCtx, key, encodeRuntimeState(state)) pipe.SAdd(decisionCtx, runtimeStateIndexKey, strconv.Itoa(channelID)) - pipe.Expire(decisionCtx, key, runtimeStateTTL) + pipe.Expire(decisionCtx, key, runtimeStateTTLFor(state, state.LastObservedAt)) _, err = pipe.Exec(decisionCtx) return err }) diff --git a/pkg/channelhealth/runtime_test.go b/pkg/channelhealth/runtime_test.go index b89fcfe7..4fb71426 100644 --- a/pkg/channelhealth/runtime_test.go +++ b/pkg/channelhealth/runtime_test.go @@ -359,6 +359,60 @@ func TestEvaluateTimeoutGuardIgnoresTimeoutObservedBeforeRecovery(t *testing.T) require.False(t, stale.State.RuntimeDisabled) } +func TestEvaluateTimeoutGuardKeepsRecoveryBoundaryAfterNewSample(t *testing.T) { + withRuntimeRedis(t) + setting := testRuntimeSetting() + setting.TimeoutAutoDisableMinimumSamples = 10 + oldEvidence := AttemptEvidence{ + ChannelID: 936, AttemptID: "initial", RequestMode: RequestModeStreaming, + TimeoutKind: TimeoutKindStreamingFirstResult, TimeoutEligible: true, AutoBan: true, + ObservedAt: time.Now().Add(-time.Second), + } + _, err := EvaluateTimeoutGuard(context.Background(), oldEvidence, setting) + require.NoError(t, err) + _, err = RestoreRuntimeState(context.Background(), 936) + require.NoError(t, err) + newSample, err := EvaluateTimeoutGuard(context.Background(), AttemptEvidence{ + ChannelID: 936, AttemptID: "new-success", RequestMode: RequestModeStreaming, + TimeoutKind: TimeoutKindFirstResponse, Success: true, AutoBan: true, ObservedAt: time.Now(), + }, setting) + require.NoError(t, err) + oldEvidence.AttemptID = "delayed-old-timeout" + stale, err := EvaluateTimeoutGuard(context.Background(), oldEvidence, setting) + require.NoError(t, err) + require.False(t, stale.Applied) + require.Equal(t, newSample.State.Generation, stale.State.Generation) + require.True(t, newSample.State.LastObservedAt.Equal(stale.State.LastObservedAt)) + require.Zero(t, stale.State.TimeoutCount[RequestModeStreaming]) + require.Zero(t, stale.State.Penalty) +} + +func TestEvaluateTimeoutGuardTTLIncludesActiveDeadlines(t *testing.T) { + for _, disabled := range []bool{false, true} { + t.Run(strconv.FormatBool(disabled), func(t *testing.T) { + withRuntimeRedis(t) + setting := testRuntimeSetting() + setting.PenaltyCooldownSeconds = 30 * 24 * 3600 + setting.TimeoutAutoDisableDurationSeconds = 30 * 24 * 3600 + setting.TimeoutAutoDisableCount = 1 + setting.TimeoutAutoDisableEnabled = disabled + result, err := EvaluateTimeoutGuard(context.Background(), AttemptEvidence{ + ChannelID: 936, AttemptID: "long-deadline", RequestMode: RequestModeStreaming, + TimeoutKind: TimeoutKindStreamingFirstResult, TimeoutEligible: true, AutoBan: true, + ObservedAt: time.Now(), + }, setting) + require.NoError(t, err) + deadline := result.State.PenaltyUntil + if disabled { + deadline = result.State.DisabledUntil + } + ttl, err := maxcommon.RDB.TTL(context.Background(), RuntimeStateKey(936)).Result() + require.NoError(t, err) + require.Greater(t, ttl, time.Until(deadline)) + }) + } +} + func TestStreamIdleTimeoutDoesNotCountOrDemote(t *testing.T) { withRuntimeRedis(t) setting := testRuntimeSetting() diff --git a/relay/channel/api_request.go b/relay/channel/api_request.go index 3e4e9ee4..8f042e2b 100644 --- a/relay/channel/api_request.go +++ b/relay/channel/api_request.go @@ -28,6 +28,13 @@ import ( "github.com/gorilla/websocket" ) +type streamingHeaderTransportKey struct { + base *http.Transport + timeout time.Duration +} + +var streamingHeaderTransportCache sync.Map + // applyUpstreamBodyMetadata restores metadata hidden when http.NewRequest // wraps an arbitrary reader in req.Body. func applyUpstreamBodyMetadata(req *http.Request, body io.Reader, info *common.RelayInfo) { @@ -596,14 +603,7 @@ func clientWithStreamingHeaderTimeout(client *http.Client, info *common.RelayInf if setting == nil || setting.StreamingFirstResultTimeoutSeconds <= 0 { return client } - remaining := common.RemainingChannelFirstResponseTimeout( - info.ChannelAttemptStartTime(), - time.Duration(setting.StreamingFirstResultTimeoutSeconds)*time.Second, - time.Now(), - ) - if remaining <= 0 { - return client - } + configured := time.Duration(setting.StreamingFirstResultTimeoutSeconds) * time.Second transport, ok := client.Transport.(*http.Transport) if client.Transport == nil { @@ -615,12 +615,21 @@ func clientWithStreamingHeaderTimeout(client *http.Client, info *common.RelayInf // would also terminate the body after a successful first response. return client } - clonedTransport := transport.Clone() - if clonedTransport.ResponseHeaderTimeout == 0 || remaining < clonedTransport.ResponseHeaderTimeout { - clonedTransport.ResponseHeaderTimeout = remaining + effective := configured + if transport.ResponseHeaderTimeout > 0 && transport.ResponseHeaderTimeout < effective { + effective = transport.ResponseHeaderTimeout } + cacheKey := streamingHeaderTransportKey{base: transport, timeout: effective} + if cached, ok := streamingHeaderTransportCache.Load(cacheKey); ok { + clone := *client + clone.Transport = cached.(*http.Transport) + return &clone + } + clonedTransport := transport.Clone() + clonedTransport.ResponseHeaderTimeout = effective + actual, _ := streamingHeaderTransportCache.LoadOrStore(cacheKey, clonedTransport) clone := *client - clone.Transport = clonedTransport + clone.Transport = actual.(*http.Transport) return &clone } diff --git a/relay/channel/api_request_test.go b/relay/channel/api_request_test.go index ca5a48ac..5ebe5db1 100644 --- a/relay/channel/api_request_test.go +++ b/relay/channel/api_request_test.go @@ -21,7 +21,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestClientWithStreamingHeaderTimeoutUsesRemainingAttemptBudget(t *testing.T) { +func TestClientWithStreamingHeaderTimeoutCachesTransportByConfiguredBudget(t *testing.T) { originalSetting := *operation_setting.GetMonitorSetting() *operation_setting.GetMonitorSetting() = operation_setting.MonitorSetting{ StreamingFirstResultTimeoutSeconds: 1, @@ -33,18 +33,17 @@ func TestClientWithStreamingHeaderTimeoutUsesRemainingAttemptBudget(t *testing.T info := &relaycommon.RelayInfo{ IsStream: true, } - recorder := httptest.NewRecorder() - ctx, _ := gin.CreateTestContext(recorder) - ctx.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil) - info.InitChannelMeta(ctx) - time.Sleep(400 * time.Millisecond) cloned := clientWithStreamingHeaderTimeout(client, info) + clonedAgain := clientWithStreamingHeaderTimeout(client, info) require.NotSame(t, client, cloned) + require.NotSame(t, client, clonedAgain) clonedTransport, ok := cloned.Transport.(*http.Transport) require.True(t, ok) - require.Greater(t, clonedTransport.ResponseHeaderTimeout, 500*time.Millisecond) - require.Less(t, clonedTransport.ResponseHeaderTimeout, 700*time.Millisecond) + clonedAgainTransport, ok := clonedAgain.Transport.(*http.Transport) + require.True(t, ok) + require.Same(t, clonedTransport, clonedAgainTransport) + require.Equal(t, time.Second, clonedTransport.ResponseHeaderTimeout) require.Equal(t, 5*time.Second, transport.ResponseHeaderTimeout) } diff --git a/relay/common/channel_health_observation_test.go b/relay/common/channel_health_observation_test.go index 1967946e..7b8515be 100644 --- a/relay/common/channel_health_observation_test.go +++ b/relay/common/channel_health_observation_test.go @@ -144,3 +144,51 @@ func TestRelayInfoChannelTestDoesNotRecordHealthEvidence(t *testing.T) { require.Zero(t, state.Penalty) require.Zero(t, state.TimeoutCount[channelhealth.RequestModeStreaming]) } + +func TestRelayInfoSuccessSamplePublishesDisableTransition(t *testing.T) { + server := miniredis.RunT(t) + oldRDB, oldEnabled := maxcommon.RDB, maxcommon.RedisEnabled + maxcommon.RDB = redis.NewClient(&redis.Options{Addr: server.Addr()}) + maxcommon.RedisEnabled = true + t.Cleanup(func() { + _ = maxcommon.RDB.Close() + maxcommon.RDB, maxcommon.RedisEnabled = oldRDB, oldEnabled + }) + + oldSetting := *operation_setting.GetMonitorSetting() + *operation_setting.GetMonitorSetting() = operation_setting.MonitorSetting{ + TimeoutAutoDisableEnabled: true, + TimeoutAutoDisableCount: 1, + TimeoutAutoDisableMinimumSamples: 2, + StreamingFirstResultTimeoutSeconds: 1, + } + t.Cleanup(func() { *operation_setting.GetMonitorSetting() = oldSetting }) + + first, err := channelhealth.EvaluateTimeoutGuard(context.Background(), channelhealth.AttemptEvidence{ + ChannelID: 936, AttemptID: "timeout-before-success", RequestMode: channelhealth.RequestModeStreaming, + TimeoutKind: channelhealth.TimeoutKindStreamingFirstResult, TimeoutEligible: true, AutoBan: true, + ObservedAt: time.Now().Add(-time.Second), + }, operation_setting.GetMonitorSetting()) + require.NoError(t, err) + require.False(t, first.State.RuntimeDisabled) + + transitions := make(chan maxcommon.ChannelHealthRuntimeTransition, 1) + maxcommon.SetChannelHealthRuntimeTransitionObserver(func(transition maxcommon.ChannelHealthRuntimeTransition) { + transitions <- transition + }) + t.Cleanup(func() { maxcommon.SetChannelHealthRuntimeTransitionObserver(nil) }) + info := &RelayInfo{ + IsStream: true, + channelAttemptStartTime: time.Now(), + ChannelMeta: &ChannelMeta{ChannelId: 936, ChannelAutoBan: true}, + } + info.recordChannelHealthSuccess() + + select { + case transition := <-transitions: + require.Equal(t, "timeout_auto_disabled", transition.Transition) + require.True(t, transition.RuntimeDisabled) + case <-time.After(time.Second): + t.Fatal("success sample did not publish its runtime transition") + } +} diff --git a/relay/common/relay_info.go b/relay/common/relay_info.go index 7b98d9e3..b0438ad7 100644 --- a/relay/common/relay_info.go +++ b/relay/common/relay_info.go @@ -941,11 +941,23 @@ func (info *RelayInfo) recordChannelHealthSuccess() { AutoBan: info.ChannelAutoBan, ObservedAt: time.Now(), } - go func() { + go func(evidence channelhealth.AttemptEvidence, setting operation_setting.MonitorSetting) { decisionContext, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() - _, _ = channelhealth.EvaluateTimeoutGuard(decisionContext, evidence, &settingSnapshot) - }() + result, err := channelhealth.EvaluateTimeoutGuard(decisionContext, evidence, &setting) + if err != nil || result.Transition == "" { + return + } + common.PublishChannelHealthRuntimeTransition(common.ChannelHealthRuntimeTransition{ + ChannelID: evidence.ChannelID, + Transition: result.Transition, + Penalty: result.State.Penalty, + PenaltyUntil: result.State.PenaltyUntil, + RuntimeDisabled: result.State.RuntimeDisabled, + DisabledUntil: result.State.DisabledUntil, + ObservedAt: result.State.LastObservedAt, + }) + }(evidence, settingSnapshot) } // RecordChannelTimeout records a confirmed timeout outcome for the current diff --git a/service/smart_ops_alert.go b/service/smart_ops_alert.go index c218af05..a5b687bb 100644 --- a/service/smart_ops_alert.go +++ b/service/smart_ops_alert.go @@ -495,6 +495,7 @@ func GetSmartOpsAlerts() []SmartOpsAlert { if strings.HasPrefix(key, "channel_timeout_") { if _, ok := runtimeKeys[key]; !ok { delete(alertsByKey, key) + delete(smartOpsAlertMonitor.active, key) } } } diff --git a/service/smart_ops_alert_test.go b/service/smart_ops_alert_test.go index 4dc6021a..aa821b99 100644 --- a/service/smart_ops_alert_test.go +++ b/service/smart_ops_alert_test.go @@ -10,9 +10,39 @@ import ( "github.com/MAX-API-Next/MAX-API/model" "github.com/MAX-API-Next/MAX-API/pkg/channelhealth" "github.com/MAX-API-Next/MAX-API/setting/operation_setting" + "github.com/alicebob/miniredis/v2" + "github.com/go-redis/redis/v8" "github.com/stretchr/testify/require" ) +func TestGetSmartOpsAlertsRemovesStaleLocalChannelAlertAfterRuntimeRead(t *testing.T) { + server := miniredis.RunT(t) + oldRDB, oldEnabled := common.RDB, common.RedisEnabled + common.RDB = redis.NewClient(&redis.Options{Addr: server.Addr()}) + common.RedisEnabled = true + t.Cleanup(func() { + _ = common.RDB.Close() + common.RDB, common.RedisEnabled = oldRDB, oldEnabled + }) + oldActive := smartOpsAlertMonitor.active + smartOpsAlertMonitor.Lock() + smartOpsAlertMonitor.active = map[string]SmartOpsAlert{ + channelHealthPriorityAlertKey(936): {Key: channelHealthPriorityAlertKey(936), Status: smartOpsAlertStatusFiring}, + } + smartOpsAlertMonitor.Unlock() + t.Cleanup(func() { + smartOpsAlertMonitor.Lock() + smartOpsAlertMonitor.active = oldActive + smartOpsAlertMonitor.Unlock() + }) + + require.Empty(t, GetSmartOpsAlerts()) + smartOpsAlertMonitor.Lock() + _, stillActive := smartOpsAlertMonitor.active[channelHealthPriorityAlertKey(936)] + smartOpsAlertMonitor.Unlock() + require.False(t, stillActive) +} + func TestChannelHealthRuntimeTransitionProjectsAndNotifies(t *testing.T) { originalSender := smartOpsAlertNotificationSender originalSetting := *operation_setting.GetMonitorSetting() From 501739b16e241cc969de296306c85caa24cba631 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Sat, 3 Oct 2026 05:57:57 +0800 Subject: [PATCH 09/12] fix: close channel health review findings --- pkg/channelhealth/runtime.go | 14 ++++- pkg/channelhealth/runtime_cursor_test.go | 29 +++++++++ relay/channel/api_request.go | 79 ++++++++++++++++++------ relay/channel/api_request_test.go | 44 +++++++++++++ service/smart_ops_alert.go | 46 +++++++++++--- service/smart_ops_alert_test.go | 46 ++++++++++++++ 6 files changed, 227 insertions(+), 31 deletions(-) create mode 100644 pkg/channelhealth/runtime_cursor_test.go diff --git a/pkg/channelhealth/runtime.go b/pkg/channelhealth/runtime.go index 42ab5613..11df510f 100644 --- a/pkg/channelhealth/runtime.go +++ b/pkg/channelhealth/runtime.go @@ -307,9 +307,16 @@ func ListRuntimeStates(ctx context.Context) ([]RuntimeState, error) { cursor, _ = strconv.ParseUint(value, 10, 64) } scanComplete := false - for { + persistCursor := func() error { + if cursor == 0 { + return nil + } + return maxcommon.RDB.Set(context.Background(), runtimeStateIndexCursorKey, strconv.FormatUint(cursor, 10), 0).Err() + } + for iterations := 0; iterations < 16; iterations++ { batch, next, err := maxcommon.RDB.Scan(ctx, cursor, runtimeStateKeyPrefix+"*", 200).Result() if err != nil { + _ = persistCursor() return nil, err } keys = append(keys, batch...) @@ -322,6 +329,11 @@ func ListRuntimeStates(ctx context.Context) ([]RuntimeState, error) { break } } + if !scanComplete { + if err := persistCursor(); err != nil { + return nil, err + } + } allMembers := make([]string, 0, len(members)+len(keys)) seenMembers := make(map[string]struct{}, len(members)+len(keys)) for _, member := range append(members, keys...) { diff --git a/pkg/channelhealth/runtime_cursor_test.go b/pkg/channelhealth/runtime_cursor_test.go new file mode 100644 index 00000000..9f7ec010 --- /dev/null +++ b/pkg/channelhealth/runtime_cursor_test.go @@ -0,0 +1,29 @@ +package channelhealth + +import ( + "context" + "strconv" + "testing" + + maxcommon "github.com/MAX-API-Next/MAX-API/common" + "github.com/stretchr/testify/require" +) + +func TestListRuntimeStatesPersistsCursorAfterBoundedLegacyScan(t *testing.T) { + withRuntimeRedis(t) + pipe := maxcommon.RDB.Pipeline() + for i := 1; i <= 6000; i++ { + pipe.HSet(context.Background(), runtimeStateKey(i), map[string]interface{}{"channel_id": strconv.Itoa(i), "penalty": "1"}) + } + _, err := pipe.Exec(context.Background()) + require.NoError(t, err) + _, err = ListRuntimeStates(context.Background()) + require.NoError(t, err) + migrated, err := maxcommon.RDB.Exists(context.Background(), runtimeStateIndexMigratedKey).Result() + require.NoError(t, err) + cursor, err := maxcommon.RDB.Get(context.Background(), runtimeStateIndexCursorKey).Result() + require.NoError(t, err) + t.Logf("migrated=%d cursor=%s", migrated, cursor) + require.Zero(t, migrated) + require.NotEmpty(t, cursor) +} diff --git a/relay/channel/api_request.go b/relay/channel/api_request.go index 8f042e2b..513a5462 100644 --- a/relay/channel/api_request.go +++ b/relay/channel/api_request.go @@ -28,12 +28,12 @@ import ( "github.com/gorilla/websocket" ) -type streamingHeaderTransportKey struct { +type responseHeaderTransportKey struct { base *http.Transport timeout time.Duration } -var streamingHeaderTransportCache sync.Map +var responseHeaderTransportCache sync.Map // applyUpstreamBodyMetadata restores metadata hidden when http.NewRequest // wraps an arbitrary reader in req.Body. @@ -560,8 +560,8 @@ func doRequest(c *gin.Context, req *http.Request, info *common.RelayInfo) (*http } if info != nil && !info.IsStream && resp.Body != nil { setting := operation_setting.GetMonitorSetting() - if setting != nil && setting.NonStreamingResponseTimeoutSeconds > 0 { - resp.Body = &timeoutTrackingBody{ReadCloser: resp.Body, info: info} + if setting != nil && setting.NonStreamingResponseTimeoutSeconds > 0 && !isServerSentEventsResponse(resp) { + resp.Body = newTimeoutTrackingBody(resp.Body, info, time.Duration(setting.NonStreamingResponseTimeoutSeconds)*time.Second) } } @@ -583,12 +583,7 @@ func clientWithNonStreamingTimeout(client *http.Client, info *common.RelayInfo) return client } timeout := time.Duration(setting.NonStreamingResponseTimeoutSeconds) * time.Second - if client.Timeout > 0 && client.Timeout <= timeout { - return client - } - clone := *client - clone.Timeout = timeout - return &clone + return clientWithResponseHeaderTimeout(client, timeout) } // clientWithStreamingHeaderTimeout bounds the wait for upstream response @@ -604,48 +599,92 @@ func clientWithStreamingHeaderTimeout(client *http.Client, info *common.RelayInf return client } configured := time.Duration(setting.StreamingFirstResultTimeoutSeconds) * time.Second + return clientWithResponseHeaderTimeout(client, configured) +} +func clientWithResponseHeaderTimeout(client *http.Client, timeout time.Duration) *http.Client { + if client == nil || timeout <= 0 { + return client + } transport, ok := client.Transport.(*http.Transport) if client.Transport == nil { transport, ok = http.DefaultTransport.(*http.Transport) } if !ok || transport == nil { - // All built-in relay clients use *http.Transport. Custom round - // trippers are left untouched because imposing http.Client.Timeout - // would also terminate the body after a successful first response. return client } - effective := configured + effective := timeout if transport.ResponseHeaderTimeout > 0 && transport.ResponseHeaderTimeout < effective { effective = transport.ResponseHeaderTimeout } - cacheKey := streamingHeaderTransportKey{base: transport, timeout: effective} - if cached, ok := streamingHeaderTransportCache.Load(cacheKey); ok { + cacheKey := responseHeaderTransportKey{base: transport, timeout: effective} + if cached, ok := responseHeaderTransportCache.Load(cacheKey); ok { clone := *client clone.Transport = cached.(*http.Transport) return &clone } clonedTransport := transport.Clone() clonedTransport.ResponseHeaderTimeout = effective - actual, _ := streamingHeaderTransportCache.LoadOrStore(cacheKey, clonedTransport) + actual, _ := responseHeaderTransportCache.LoadOrStore(cacheKey, clonedTransport) clone := *client clone.Transport = actual.(*http.Transport) return &clone } +func isServerSentEventsResponse(resp *http.Response) bool { + return resp != nil && strings.HasPrefix(strings.ToLower(strings.TrimSpace(resp.Header.Get("Content-Type"))), "text/event-stream") +} + type timeoutTrackingBody struct { io.ReadCloser - info *common.RelayInfo + info *common.RelayInfo + timer *time.Timer + timedOut atomic.Bool + recordOnce sync.Once +} + +func newTimeoutTrackingBody(body io.ReadCloser, info *common.RelayInfo, timeout time.Duration) *timeoutTrackingBody { + tracked := &timeoutTrackingBody{ReadCloser: body, info: info} + if timeout > 0 { + tracked.timer = time.AfterFunc(timeout, tracked.expire) + } + return tracked +} + +func (body *timeoutTrackingBody) expire() { + body.timedOut.Store(true) + body.recordTimeout() + _ = body.ReadCloser.Close() +} + +func (body *timeoutTrackingBody) recordTimeout() { + body.recordOnce.Do(func() { + if body.info != nil { + body.info.RecordChannelTimeout("request_timeout") + } + }) } func (body *timeoutTrackingBody) Read(p []byte) (int, error) { n, err := body.ReadCloser.Read(p) - if err != nil && isUpstreamTimeout(err) && body.info != nil { - body.info.RecordChannelTimeout("request_timeout") + if err != nil { + if body.timer != nil { + body.timer.Stop() + } + if body.timedOut.Load() || isUpstreamTimeout(err) { + body.recordTimeout() + } } return n, err } +func (body *timeoutTrackingBody) Close() error { + if body.timer != nil { + body.timer.Stop() + } + return body.ReadCloser.Close() +} + func isUpstreamTimeout(err error) bool { if err == nil { return false diff --git a/relay/channel/api_request_test.go b/relay/channel/api_request_test.go index 5ebe5db1..8448b603 100644 --- a/relay/channel/api_request_test.go +++ b/relay/channel/api_request_test.go @@ -47,6 +47,50 @@ func TestClientWithStreamingHeaderTimeoutCachesTransportByConfiguredBudget(t *te require.Equal(t, 5*time.Second, transport.ResponseHeaderTimeout) } +func TestClientWithNonStreamingTimeoutUsesHeaderBudgetWithoutShorteningBodyClient(t *testing.T) { + originalSetting := *operation_setting.GetMonitorSetting() + *operation_setting.GetMonitorSetting() = operation_setting.MonitorSetting{ + NonStreamingResponseTimeoutSeconds: 1, + } + t.Cleanup(func() { *operation_setting.GetMonitorSetting() = originalSetting }) + + transport := &http.Transport{ResponseHeaderTimeout: 5 * time.Second} + client := &http.Client{Transport: transport, Timeout: 30 * time.Second} + cloned := clientWithNonStreamingTimeout(client, &relaycommon.RelayInfo{}) + require.NotSame(t, client, cloned) + require.Equal(t, 30*time.Second, cloned.Timeout) + clonedTransport, ok := cloned.Transport.(*http.Transport) + require.True(t, ok) + require.Equal(t, time.Second, clonedTransport.ResponseHeaderTimeout) + require.Equal(t, 5*time.Second, transport.ResponseHeaderTimeout) +} + +func TestTimeoutTrackingBodyEnforcesDeadline(t *testing.T) { + reader, writer := io.Pipe() + tracked := newTimeoutTrackingBody(reader, nil, 20*time.Millisecond) + t.Cleanup(func() { _ = tracked.Close(); _ = writer.Close() }) + + result := make(chan error, 1) + go func() { + _, err := tracked.Read(make([]byte, 1)) + result <- err + }() + + select { + case err := <-result: + require.Error(t, err) + case <-time.After(time.Second): + t.Fatal("timeout tracking body did not stop a stalled read") + } +} + +func TestServerSentEventsResponseSkipsNonStreamingBodyTimeout(t *testing.T) { + sse := &http.Response{Header: http.Header{"Content-Type": []string{"text/event-stream; charset=utf-8"}}} + json := &http.Response{Header: http.Header{"Content-Type": []string{"application/json"}}} + require.True(t, isServerSentEventsResponse(sse)) + require.False(t, isServerSentEventsResponse(json)) +} + type taskHeaderAdaptor struct { taskcommon.BaseBilling url string diff --git a/service/smart_ops_alert.go b/service/smart_ops_alert.go index a5b687bb..94777d9e 100644 --- a/service/smart_ops_alert.go +++ b/service/smart_ops_alert.go @@ -403,6 +403,7 @@ func handleChannelHealthRuntimeTransition(transition common.ChannelHealthRuntime // Recovery must close both possible channel projections. The critical // disable alert may have been created before a process restart, while a // warning can still exist when an operator restores a penalty directly. + disabledAlertWasActive := channelHealthAlertWasActive(channelHealthDisabledAlertKey(transition.ChannelID)) projectSmartOpsActiveAlert(SmartOpsAlert{ Key: channelHealthDisabledAlertKey(transition.ChannelID), Status: smartOpsAlertStatusResolved, @@ -411,16 +412,18 @@ func handleChannelHealthRuntimeTransition(transition common.ChannelHealthRuntime Key: channelHealthPriorityAlertKey(transition.ChannelID), Status: smartOpsAlertStatusResolved, }) - if setting := operation_setting.GetMonitorSetting(); setting != nil && setting.ChannelTimeoutNotificationEnabled { - smartOpsAlertNotificationSender(SmartOpsAlert{ - Key: channelHealthDisabledAlertKey(transition.ChannelID), - Status: smartOpsAlertStatusResolved, - Severity: smartOpsAlertSeverityCritical, - Component: "channel", - Node: smartOpsAlertNodeName(), - ObservedAt: observedAt, - Message: fmt.Sprintf("渠道 %d 已恢复运行态，告警已关闭", transition.ChannelID), - }) + if disabledAlertWasActive { + if setting := operation_setting.GetMonitorSetting(); setting != nil && setting.ChannelTimeoutNotificationEnabled { + smartOpsAlertNotificationSender(SmartOpsAlert{ + Key: channelHealthDisabledAlertKey(transition.ChannelID), + Status: smartOpsAlertStatusResolved, + Severity: smartOpsAlertSeverityCritical, + Component: "channel", + Node: smartOpsAlertNodeName(), + ObservedAt: observedAt, + Message: fmt.Sprintf("渠道 %d 已恢复运行态，告警已关闭", transition.ChannelID), + }) + } } return default: @@ -437,6 +440,29 @@ func handleChannelHealthRuntimeTransition(transition common.ChannelHealthRuntime } } +// channelHealthAlertWasActive checks the process-local projection first, then +// the shared Redis projection. This preserves a resolved notification after a +// process restart without creating a false recovery notification for a channel +// that only had a priority warning. +func channelHealthAlertWasActive(key string) bool { + smartOpsAlertMonitor.Lock() + alert, ok := smartOpsAlertMonitor.active[key] + smartOpsAlertMonitor.Unlock() + if ok && alert.Status == smartOpsAlertStatusFiring { + return true + } + runtimeAlerts, available := loadChannelHealthSmartOpsAlerts() + if !available { + return false + } + for _, runtimeAlert := range runtimeAlerts { + if runtimeAlert.Key == key && runtimeAlert.Status == smartOpsAlertStatusFiring { + return true + } + } + return false +} + func formatChannelHealthPriorityAlertMessage(transition common.ChannelHealthRuntimeTransition) string { message := fmt.Sprintf("渠道 %d 已因上游超时降低运行态优先级，扣减 %d", transition.ChannelID, transition.Penalty) if !transition.PenaltyUntil.IsZero() { diff --git a/service/smart_ops_alert_test.go b/service/smart_ops_alert_test.go index aa821b99..4a0de89d 100644 --- a/service/smart_ops_alert_test.go +++ b/service/smart_ops_alert_test.go @@ -119,6 +119,52 @@ func TestChannelHealthRuntimeTransitionProjectsAndNotifies(t *testing.T) { } } +func TestChannelHealthRecoveryDoesNotNotifyPriorityOnlyAlert(t *testing.T) { + originalSender := smartOpsAlertNotificationSender + originalSetting := *operation_setting.GetMonitorSetting() + originalRedisEnabled := common.RedisEnabled + smartOpsAlertMonitor.Lock() + originalActive := smartOpsAlertMonitor.active + smartOpsAlertMonitor.active = map[string]SmartOpsAlert{ + channelHealthPriorityAlertKey(936): { + Key: channelHealthPriorityAlertKey(936), + Status: smartOpsAlertStatusFiring, + }, + } + smartOpsAlertMonitor.Unlock() + t.Cleanup(func() { + smartOpsAlertNotificationSender = originalSender + *operation_setting.GetMonitorSetting() = originalSetting + common.RedisEnabled = originalRedisEnabled + smartOpsAlertMonitor.Lock() + smartOpsAlertMonitor.active = originalActive + smartOpsAlertMonitor.Unlock() + }) + + common.RedisEnabled = false + *operation_setting.GetMonitorSetting() = operation_setting.MonitorSetting{ + ChannelTimeoutNotificationEnabled: true, + } + var events []SmartOpsAlert + smartOpsAlertNotificationSender = func(alert SmartOpsAlert) { + events = append(events, alert) + } + + handleChannelHealthRuntimeTransition(common.ChannelHealthRuntimeTransition{ + ChannelID: 936, + Transition: "runtime_recovered", + ObservedAt: time.Now(), + }) + + require.Empty(t, events) + smartOpsAlertMonitor.Lock() + defer smartOpsAlertMonitor.Unlock() + _, priorityStillActive := smartOpsAlertMonitor.active[channelHealthPriorityAlertKey(936)] + _, disabledStillActive := smartOpsAlertMonitor.active[channelHealthDisabledAlertKey(936)] + require.False(t, priorityStillActive) + require.False(t, disabledStillActive) +} + func TestChannelHealthTimeoutCountUsesTheLatestModeForSameModePolicy(t *testing.T) { state := channelhealth.RuntimeState{ TimeoutCount: map[string]int64{ From 4c0340407c4a6fd032ddc1efb316857abdecdde7 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Sat, 3 Oct 2026 06:44:19 +0800 Subject: [PATCH 10/12] fix: close final channel health review findings --- pkg/channelhealth/runtime.go | 3 +++ pkg/channelhealth/runtime_test.go | 26 +++++++++++++++++++ relay/channel/api_request.go | 7 ++++- relay/channel/claude/relay-claude.go | 9 +++++++ relay/channel/claude/relay_claude_test.go | 23 ++++++++++++++++ relay/channel/openai/chat_stream_test.go | 24 +++++++++++++++++ relay/channel/openai/relay-openai.go | 5 ++++ .../smart-ops/channel-health/index.tsx | 19 +++++++++++++- web/default/src/i18n/locales/en.json | 6 +++++ web/default/src/i18n/locales/fr.json | 6 +++++ web/default/src/i18n/locales/ja.json | 6 +++++ web/default/src/i18n/locales/ru.json | 6 +++++ web/default/src/i18n/locales/vi.json | 6 +++++ web/default/src/i18n/locales/zh.json | 6 +++++ web/default/src/i18n/static-keys.ts | 9 +++++++ 15 files changed, 159 insertions(+), 2 deletions(-) diff --git a/pkg/channelhealth/runtime.go b/pkg/channelhealth/runtime.go index 11df510f..51000baf 100644 --- a/pkg/channelhealth/runtime.go +++ b/pkg/channelhealth/runtime.go @@ -538,6 +538,9 @@ func EvaluateTimeoutGuard(ctx context.Context, evidence AttemptEvidence, setting result.Applied = true } } + if !shouldCount && !shouldRecordSample && !result.Applied { + return nil + } state.Generation++ state.LastEvent = evidence.TimeoutKind diff --git a/pkg/channelhealth/runtime_test.go b/pkg/channelhealth/runtime_test.go index 4fb71426..f55029b0 100644 --- a/pkg/channelhealth/runtime_test.go +++ b/pkg/channelhealth/runtime_test.go @@ -78,6 +78,32 @@ func TestEvaluateTimeoutGuardAppliesPenaltyAndDeduplicates(t *testing.T) { require.Equal(t, int64(1), duplicate.State.TimeoutCount[RequestModeStreaming]) } +func TestEvaluateTimeoutGuardSkipsInactiveEvidenceWrite(t *testing.T) { + withRuntimeRedis(t) + setting := testRuntimeSetting() + setting.StreamingFirstResultTimeoutSeconds = 0 + + result, err := EvaluateTimeoutGuard(context.Background(), AttemptEvidence{ + ChannelID: 936, + AttemptID: "inactive-stream-success", + RequestMode: RequestModeStreaming, + TimeoutKind: TimeoutKindFirstResponse, + Success: true, + AutoBan: true, + ObservedAt: time.Now(), + }, setting) + require.NoError(t, err) + require.False(t, result.Applied) + require.False(t, result.Deduplicated) + + exists, err := maxcommon.RDB.Exists(context.Background(), RuntimeStateKey(936)).Result() + require.NoError(t, err) + require.Zero(t, exists) + indexed, err := maxcommon.RDB.SIsMember(context.Background(), runtimeStateIndexKey, "936").Result() + require.NoError(t, err) + require.False(t, indexed) +} + func TestEvaluateTimeoutGuardRetriesConcurrentRedisTransactions(t *testing.T) { withRuntimeRedis(t) setting := testRuntimeSetting() diff --git a/relay/channel/api_request.go b/relay/channel/api_request.go index 513a5462..025cdf23 100644 --- a/relay/channel/api_request.go +++ b/relay/channel/api_request.go @@ -561,7 +561,12 @@ func doRequest(c *gin.Context, req *http.Request, info *common.RelayInfo) (*http if info != nil && !info.IsStream && resp.Body != nil { setting := operation_setting.GetMonitorSetting() if setting != nil && setting.NonStreamingResponseTimeoutSeconds > 0 && !isServerSentEventsResponse(resp) { - resp.Body = newTimeoutTrackingBody(resp.Body, info, time.Duration(setting.NonStreamingResponseTimeoutSeconds)*time.Second) + budget := common.RemainingChannelFirstResponseTimeout( + info.ChannelAttemptStartTime(), + time.Duration(setting.NonStreamingResponseTimeoutSeconds)*time.Second, + time.Now(), + ) + resp.Body = newTimeoutTrackingBody(resp.Body, info, budget) } } diff --git a/relay/channel/claude/relay-claude.go b/relay/channel/claude/relay-claude.go index 4474de90..8de8b3d8 100644 --- a/relay/channel/claude/relay-claude.go +++ b/relay/channel/claude/relay-claude.go @@ -941,6 +941,15 @@ func ClaudeStreamHandler(c *gin.Context, resp *http.Response, info *relaycommon. if err != nil { return nil, err } + if info.StreamStatus != nil && + info.StreamStatus.EndReason == relaycommon.StreamEndReasonTimeout && + !info.HasRecordedChannelFirstResponse() { + return nil, types.NewOpenAIError( + fmt.Errorf("upstream Claude stream timed out before the first response"), + types.ErrorCodeChannelResponseTimeExceeded, + http.StatusRequestTimeout, + ) + } HandleStreamFinalResponse(c, info, claudeInfo) if service.ResponseAuditEnabled() { diff --git a/relay/channel/claude/relay_claude_test.go b/relay/channel/claude/relay_claude_test.go index 05a87279..0a45fb30 100644 --- a/relay/channel/claude/relay_claude_test.go +++ b/relay/channel/claude/relay_claude_test.go @@ -2,6 +2,8 @@ package claude import ( "encoding/base64" + "io" + "net/http" "net/http/httptest" "strings" "testing" @@ -20,6 +22,27 @@ func commonPointer[T any](value T) *T { return &value } +func TestClaudeStreamHandlerReturnsRetryableErrorBeforeFirstResponseTimeout(t *testing.T) { + gin.SetMode(gin.TestMode) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/messages", nil) + info := &relaycommon.RelayInfo{ + IsStream: true, + RelayFormat: types.RelayFormatClaude, + ChannelMeta: &relaycommon.ChannelMeta{UpstreamModelName: "claude-test"}, + DisablePing: true, + } + info.InitChannelMeta(c) + info.StreamStatus = relaycommon.NewStreamStatus() + info.StreamStatus.SetEndReason(relaycommon.StreamEndReasonTimeout, nil) + + usage, err := ClaudeStreamHandler(c, &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(""))}, info) + require.Nil(t, usage) + require.NotNil(t, err) + require.Equal(t, types.ErrorCodeChannelResponseTimeExceeded, err.GetErrorCode()) + require.False(t, types.IsSkipRetryError(err)) +} + func setClaudeToolPricesForTest(t *testing.T, additions map[string]float64) { t.Helper() setting := config.GlobalConfig.Get("tool_price_setting").(*operation_setting.ToolPriceSetting) diff --git a/relay/channel/openai/chat_stream_test.go b/relay/channel/openai/chat_stream_test.go index 7b0f2231..b1d9ad38 100644 --- a/relay/channel/openai/chat_stream_test.go +++ b/relay/channel/openai/chat_stream_test.go @@ -241,6 +241,30 @@ func TestNativeChatStreamFlushesFirstPayloadWithoutWaitingForAnotherFrame(t *tes require.Contains(t, recorder.Body.String(), "first") } +func TestOaiStreamHandlerCountsBufferedFirstChunkAsFirstResponse(t *testing.T) { + originalSetting := *operation_setting.GetMonitorSetting() + *operation_setting.GetMonitorSetting() = operation_setting.MonitorSetting{ + StreamingFirstResultTimeoutSeconds: 1, + } + t.Cleanup(func() { *operation_setting.GetMonitorSetting() = originalSetting }) + + c, recorder, info := newChatStreamTest(t) + info.RelayMode = relayconstant.RelayModeCompletions + reader, writer := io.Pipe() + t.Cleanup(func() { _ = reader.Close(); _ = writer.Close() }) + go func() { + _, _ = io.WriteString(writer, "data: {\"id\":\"cmpl-1\",\"object\":\"text_completion\",\"model\":\"gpt-test\",\"choices\":[{\"text\":\"first\",\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}\n\n") + time.Sleep(1200 * time.Millisecond) + _ = writer.Close() + }() + + _, apiErr := OaiStreamHandler(c, info, &http.Response{StatusCode: http.StatusOK, Body: reader}) + require.Nil(t, apiErr) + require.True(t, info.HasRecordedChannelFirstResponse()) + require.NotEqual(t, relaycommon.StreamEndReasonTimeout, info.StreamStatus.EndReason) + require.Contains(t, recorder.Body.String(), "first") +} + func TestNativeChatStreamPingNeverAllowsRetryAfterCommit(t *testing.T) { c, _, info := newChatStreamTest(t) info.DisablePing = false diff --git a/relay/channel/openai/relay-openai.go b/relay/channel/openai/relay-openai.go index 80f6b01a..70fc51fa 100644 --- a/relay/channel/openai/relay-openai.go +++ b/relay/channel/openai/relay-openai.go @@ -149,6 +149,11 @@ func OaiStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Re if err := processTokenData(info.RelayMode, data, &responseTextBuilder, &toolCount); err != nil { logger.LogError(c, "error processing stream token data: "+err.Error()) sr.Error(err) + } else { + // This handler retains the latest event so the terminal usage frame + // can be handled separately. Treat the accepted first event as the + // first result so the scanner does not wait for a second frame. + sr.MarkDelivered() } } }) diff --git a/web/default/src/features/smart-ops/channel-health/index.tsx b/web/default/src/features/smart-ops/channel-health/index.tsx index 445d214d..8cab67fe 100644 --- a/web/default/src/features/smart-ops/channel-health/index.tsx +++ b/web/default/src/features/smart-ops/channel-health/index.tsx @@ -66,6 +66,16 @@ import type { SmartOpsAlert } from '../types' const ALERT_POLL_INTERVAL_MS = 30000 +const DELIVERY_STATUS_LABELS: Record = { + queued: 'Queued', + sending: 'Sending', + sent: 'Sent', + skipped_repeat: 'Skipped (repeat interval)', + skipped_unconfigured: 'Skipped (not configured)', + failed: 'Delivery failed', + delivery_unknown: 'Delivery status unknown', +} + const DEFAULT_OPTION_VALUES = { 'monitor_setting.auto_priority_demotion_enabled': false, 'monitor_setting.streaming_first_result_timeout_seconds': 0, @@ -933,7 +943,14 @@ export function ChannelHealth(): ReactElement { {alert.delivery_status && (
- {t('Delivery status')}: {alert.delivery_status} + {t('Delivery status')}:{' '} + {DELIVERY_STATUS_LABELS[alert.delivery_status] + ? t( + DELIVERY_STATUS_LABELS[ + alert.delivery_status + ] + ) + : alert.delivery_status} {alert.delivery_attempts != null ? ` (${alert.delivery_attempts})` : ''} diff --git a/web/default/src/i18n/locales/en.json b/web/default/src/i18n/locales/en.json index bcbb717e..fd128bd8 100644 --- a/web/default/src/i18n/locales/en.json +++ b/web/default/src/i18n/locales/en.json @@ -5622,6 +5622,12 @@ "Successful probes required before automatic recovery.": "Successful probes required before automatic recovery.", "Channel timeout notifications": "Channel timeout notifications", "Delivery status": "Delivery status", + "Sending": "Sending", + "Sent": "Sent", + "Skipped (repeat interval)": "Skipped (repeat interval)", + "Skipped (not configured)": "Skipped (not configured)", + "Delivery failed": "Delivery failed", + "Delivery status unknown": "Delivery status unknown", "Recovery mode": "Recovery mode", "Automatic": "Automatic", "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.", diff --git a/web/default/src/i18n/locales/fr.json b/web/default/src/i18n/locales/fr.json index 24929976..128cc654 100644 --- a/web/default/src/i18n/locales/fr.json +++ b/web/default/src/i18n/locales/fr.json @@ -5762,6 +5762,12 @@ "Critical": "Critique", "Channel timeout notifications": "Notifications de dépassement du canal", "Delivery status": "État de livraison", + "Sending": "Envoi en cours", + "Sent": "Envoyé", + "Skipped (repeat interval)": "Ignoré (intervalle de répétition)", + "Skipped (not configured)": "Ignoré (non configuré)", + "Delivery failed": "Échec de la livraison", + "Delivery status unknown": "État de livraison inconnu", "Recovery mode": "Mode de récupération", "Automatic": "Automatique", "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "La récupération automatique lève la pénalité après son délai de refroidissement ; la récupération manuelle la conserve jusqu’à l’intervention d’un administrateur.", diff --git a/web/default/src/i18n/locales/ja.json b/web/default/src/i18n/locales/ja.json index a9cfc308..6c46327e 100644 --- a/web/default/src/i18n/locales/ja.json +++ b/web/default/src/i18n/locales/ja.json @@ -5762,6 +5762,12 @@ "Critical": "重大", "Channel timeout notifications": "チャネルタイムアウト通知", "Delivery status": "配信状態", + "Sending": "送信中", + "Sent": "送信済み", + "Skipped (repeat interval)": "スキップ（繰り返し間隔）", + "Skipped (not configured)": "スキップ（未設定）", + "Delivery failed": "配信失敗", + "Delivery status unknown": "配信状態不明", "Recovery mode": "復旧モード", "Automatic": "自動", "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "自動復旧はクールダウン後にペナルティを解除し、手動復旧では管理者が操作するまでペナルティを維持します。", diff --git a/web/default/src/i18n/locales/ru.json b/web/default/src/i18n/locales/ru.json index c1ebaec9..edf25f9e 100644 --- a/web/default/src/i18n/locales/ru.json +++ b/web/default/src/i18n/locales/ru.json @@ -5770,6 +5770,12 @@ "Critical": "Критический", "Channel timeout notifications": "Уведомления о таймаутах канала", "Delivery status": "Состояние доставки", + "Sending": "Отправка", + "Sent": "Отправлено", + "Skipped (repeat interval)": "Пропущено (интервал повторов)", + "Skipped (not configured)": "Пропущено (не настроено)", + "Delivery failed": "Ошибка доставки", + "Delivery status unknown": "Состояние доставки неизвестно", "Recovery mode": "Режим восстановления", "Automatic": "Автоматический", "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "Автоматическое восстановление снимает штраф после периода охлаждения; при ручном восстановлении штраф сохраняется до действий администратора.", diff --git a/web/default/src/i18n/locales/vi.json b/web/default/src/i18n/locales/vi.json index a7a21eaf..8a22f8b9 100644 --- a/web/default/src/i18n/locales/vi.json +++ b/web/default/src/i18n/locales/vi.json @@ -5762,6 +5762,12 @@ "Critical": "Nghiêm trọng", "Channel timeout notifications": "Thông báo thời gian chờ của kênh", "Delivery status": "Trạng thái gửi", + "Sending": "Đang gửi", + "Sent": "Đã gửi", + "Skipped (repeat interval)": "Đã bỏ qua (khoảng lặp)", + "Skipped (not configured)": "Đã bỏ qua (chưa cấu hình)", + "Delivery failed": "Gửi thất bại", + "Delivery status unknown": "Không rõ trạng thái gửi", "Recovery mode": "Chế độ khôi phục", "Automatic": "Tự động", "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "Khôi phục tự động xóa mức phạt sau thời gian nguội; khôi phục thủ công giữ mức phạt cho đến khi quản trị viên thao tác.", diff --git a/web/default/src/i18n/locales/zh.json b/web/default/src/i18n/locales/zh.json index b81b6238..52c8826d 100644 --- a/web/default/src/i18n/locales/zh.json +++ b/web/default/src/i18n/locales/zh.json @@ -5622,6 +5622,12 @@ "Successful probes required before automatic recovery.": "自动恢复前要求的成功探测次数。", "Channel timeout notifications": "渠道超时通知", "Delivery status": "投递状态", + "Sending": "发送中", + "Sent": "已发送", + "Skipped (repeat interval)": "已跳过（重复间隔）", + "Skipped (not configured)": "已跳过（未配置）", + "Delivery failed": "投递失败", + "Delivery status unknown": "投递状态未知", "Recovery mode": "恢复模式", "Automatic": "自动", "Automatic recovery restores a penalty after its cooldown; manual recovery keeps the penalty until an administrator acts.": "自动恢复会在冷却时间后解除惩罚；手动恢复会保留惩罚，直到管理员操作。", diff --git a/web/default/src/i18n/static-keys.ts b/web/default/src/i18n/static-keys.ts index 1b0eb001..3b421e85 100644 --- a/web/default/src/i18n/static-keys.ts +++ b/web/default/src/i18n/static-keys.ts @@ -86,6 +86,15 @@ export const STATIC_I18N_KEYS = [ // Billing statuses 'Paid - Needs Reconciliation', + // SmartOps alert delivery statuses (dynamic labels) + 'Queued', + 'Sending', + 'Sent', + 'Skipped (repeat interval)', + 'Skipped (not configured)', + 'Delivery failed', + 'Delivery status unknown', + // API Keys 'Enabled', 'Disabled', From 6b3e6e641a5431d7c8af52edcca23c29853ea139 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Sat, 3 Oct 2026 07:33:18 +0800 Subject: [PATCH 11/12] fix: harden channel timeout retries and caches --- common/channel_health_observation.go | 3 +- common/channel_health_observation_test.go | 20 ++++++++ relay/channel/api_request.go | 14 ++++++ relay/channel/api_request_test.go | 23 ++++++++++ relay/channel/baidu/relay-baidu.go | 3 ++ relay/channel/claude/relay-claude.go | 10 +--- relay/channel/dify/relay-dify.go | 3 ++ relay/channel/gemini/relay-gemini.go | 3 ++ relay/channel/openai/chat_via_responses.go | 4 ++ relay/channel/openai/relay-openai.go | 4 ++ relay/channel/openai/relay_responses.go | 4 ++ relay/channel/openai/responses_via_chat.go | 4 ++ relay/channel/xai/text.go | 3 ++ .../common/channel_health_observation_test.go | 7 +++ relay/common/relay_info.go | 5 ++ relay/helper/first_result_timeout.go | 26 +++++++++++ relay/helper/first_result_timeout_test.go | 30 ++++++++++++ service/http_client.go | 46 ++++++++++++++----- service/http_client_test.go | 31 +++++++++++++ 19 files changed, 222 insertions(+), 21 deletions(-) create mode 100644 relay/helper/first_result_timeout.go create mode 100644 relay/helper/first_result_timeout_test.go diff --git a/common/channel_health_observation.go b/common/channel_health_observation.go index eb8fc1dc..6fdade90 100644 --- a/common/channel_health_observation.go +++ b/common/channel_health_observation.go @@ -38,7 +38,7 @@ func startChannelHealthObservationWorker() { // It never waits for Redis and drops the observation when the bounded queue is // full, preserving the upstream response path as fail-open. func EnqueueChannelHealthObservation(observation ChannelHealthObservation) { - if observation.ChannelID <= 0 || observation.Event == "" || !RedisEnabled || RDB == nil { + if !observation.PolicyEnabled || observation.ChannelID <= 0 || observation.Event == "" || !RedisEnabled || RDB == nil { return } startChannelHealthObservationWorker() @@ -56,6 +56,7 @@ func EnqueueChannelHealthObservation(observation ChannelHealthObservation) { type ChannelHealthObservation struct { ChannelID int Event string + PolicyEnabled bool ObservedAt time.Time AttemptLatencyMS int64 Stream bool diff --git a/common/channel_health_observation_test.go b/common/channel_health_observation_test.go index 18b1791f..ed96b9d9 100644 --- a/common/channel_health_observation_test.go +++ b/common/channel_health_observation_test.go @@ -83,3 +83,23 @@ func TestRecordChannelHealthObservationSkipsRedisWhenDisabled(t *testing.T) { Event: "first_response", })) } + +func TestEnqueueChannelHealthObservationSkipsDisabledPolicy(t *testing.T) { + server := miniredis.RunT(t) + oldRDB, oldEnabled := RDB, RedisEnabled + RDB = redis.NewClient(&redis.Options{Addr: server.Addr()}) + RedisEnabled = true + t.Cleanup(func() { + _ = RDB.Close() + RDB, RedisEnabled = oldRDB, oldEnabled + }) + + EnqueueChannelHealthObservation(ChannelHealthObservation{ + ChannelID: 937, + Event: "first_response", + }) + time.Sleep(50 * time.Millisecond) + count, err := RDB.Exists(context.Background(), channelHealthRuntimeKey(937)).Result() + require.NoError(t, err) + require.Zero(t, count) +} diff --git a/relay/channel/api_request.go b/relay/channel/api_request.go index 025cdf23..bd196804 100644 --- a/relay/channel/api_request.go +++ b/relay/channel/api_request.go @@ -35,6 +35,20 @@ type responseHeaderTransportKey struct { var responseHeaderTransportCache sync.Map +func init() { + service.RegisterProxyClientCacheResetHook(resetResponseHeaderTransportCache) +} + +func resetResponseHeaderTransportCache() { + responseHeaderTransportCache.Range(func(key, value interface{}) bool { + if transport, ok := value.(*http.Transport); ok && transport != nil { + transport.CloseIdleConnections() + } + responseHeaderTransportCache.Delete(key) + return true + }) +} + // applyUpstreamBodyMetadata restores metadata hidden when http.NewRequest // wraps an arbitrary reader in req.Body. func applyUpstreamBodyMetadata(req *http.Request, body io.Reader, info *common.RelayInfo) { diff --git a/relay/channel/api_request_test.go b/relay/channel/api_request_test.go index 8448b603..689c0392 100644 --- a/relay/channel/api_request_test.go +++ b/relay/channel/api_request_test.go @@ -47,6 +47,29 @@ func TestClientWithStreamingHeaderTimeoutCachesTransportByConfiguredBudget(t *te require.Equal(t, 5*time.Second, transport.ResponseHeaderTimeout) } +func TestResetProxyClientCacheClearsDerivedResponseHeaderTransports(t *testing.T) { + originalSetting := *operation_setting.GetMonitorSetting() + *operation_setting.GetMonitorSetting() = operation_setting.MonitorSetting{ + StreamingFirstResultTimeoutSeconds: 1, + } + t.Cleanup(func() { *operation_setting.GetMonitorSetting() = originalSetting }) + t.Cleanup(service.ResetProxyClientCache) + + transport := &http.Transport{} + client := &http.Client{Transport: transport} + info := &relaycommon.RelayInfo{IsStream: true} + first := clientWithStreamingHeaderTimeout(client, info) + firstTransport, ok := first.Transport.(*http.Transport) + require.True(t, ok) + + service.ResetProxyClientCache() + second := clientWithStreamingHeaderTimeout(client, info) + secondTransport, ok := second.Transport.(*http.Transport) + require.True(t, ok) + require.NotSame(t, firstTransport, secondTransport) + require.Equal(t, time.Second, secondTransport.ResponseHeaderTimeout) +} + func TestClientWithNonStreamingTimeoutUsesHeaderBudgetWithoutShorteningBodyClient(t *testing.T) { originalSetting := *operation_setting.GetMonitorSetting() *operation_setting.GetMonitorSetting() = operation_setting.MonitorSetting{ diff --git a/relay/channel/baidu/relay-baidu.go b/relay/channel/baidu/relay-baidu.go index af6f81b3..a1bf9d19 100644 --- a/relay/channel/baidu/relay-baidu.go +++ b/relay/channel/baidu/relay-baidu.go @@ -130,6 +130,9 @@ func baiduStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http. } }) service.CloseResponseBodyGracefully(resp) + if apiErr := helper.FirstResultTimeoutError(info); apiErr != nil { + return apiErr, nil + } return nil, usage } diff --git a/relay/channel/claude/relay-claude.go b/relay/channel/claude/relay-claude.go index 8de8b3d8..db8c00bf 100644 --- a/relay/channel/claude/relay-claude.go +++ b/relay/channel/claude/relay-claude.go @@ -941,14 +941,8 @@ func ClaudeStreamHandler(c *gin.Context, resp *http.Response, info *relaycommon. if err != nil { return nil, err } - if info.StreamStatus != nil && - info.StreamStatus.EndReason == relaycommon.StreamEndReasonTimeout && - !info.HasRecordedChannelFirstResponse() { - return nil, types.NewOpenAIError( - fmt.Errorf("upstream Claude stream timed out before the first response"), - types.ErrorCodeChannelResponseTimeExceeded, - http.StatusRequestTimeout, - ) + if apiErr := helper.FirstResultTimeoutError(info); apiErr != nil { + return nil, apiErr } HandleStreamFinalResponse(c, info, claudeInfo) diff --git a/relay/channel/dify/relay-dify.go b/relay/channel/dify/relay-dify.go index c91fe4d4..83d15d5d 100644 --- a/relay/channel/dify/relay-dify.go +++ b/relay/channel/dify/relay-dify.go @@ -252,6 +252,9 @@ func difyStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.R sr.Error(err) } }) + if apiErr := helper.FirstResultTimeoutError(info); apiErr != nil { + return nil, apiErr + } helper.Done(c) if usage.TotalTokens == 0 { usage = service.ResponseText2Usage(c, responseText, info.UpstreamModelName, info.GetEstimatePromptTokens()) diff --git a/relay/channel/gemini/relay-gemini.go b/relay/channel/gemini/relay-gemini.go index 95b999e5..42f2c3eb 100644 --- a/relay/channel/gemini/relay-gemini.go +++ b/relay/channel/gemini/relay-gemini.go @@ -1520,6 +1520,9 @@ func geminiStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http sr.Stop(fmt.Errorf("gemini callback stopped")) } }) + if apiErr := helper.FirstResultTimeoutError(info); apiErr != nil { + return nil, apiErr + } patchGeminiZeroCompletionUsage(c, info, usage, responseText.String(), imageCount) diff --git a/relay/channel/openai/chat_via_responses.go b/relay/channel/openai/chat_via_responses.go index abfc7adc..73eb61b4 100644 --- a/relay/channel/openai/chat_via_responses.go +++ b/relay/channel/openai/chat_via_responses.go @@ -586,6 +586,10 @@ func OaiResponsesToChatStreamHandler(c *gin.Context, info *relaycommon.RelayInfo } }) + if apiErr := helper.FirstResultTimeoutError(info); apiErr != nil { + return nil, apiErr + } + if streamErr != nil { return nil, streamErr } diff --git a/relay/channel/openai/relay-openai.go b/relay/channel/openai/relay-openai.go index 70fc51fa..42fca63f 100644 --- a/relay/channel/openai/relay-openai.go +++ b/relay/channel/openai/relay-openai.go @@ -159,6 +159,10 @@ func OaiStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Re }) // 对音频模型，从倒数第二个stream data中提取usage信息 + if apiErr := helper.FirstResultTimeoutError(info); apiErr != nil { + return nil, apiErr + } + if isAudioModel && secondLastStreamData != "" { var streamResp struct { Usage *dto.Usage `json:"usage"` diff --git a/relay/channel/openai/relay_responses.go b/relay/channel/openai/relay_responses.go index 4c4a34ba..17f2206b 100644 --- a/relay/channel/openai/relay_responses.go +++ b/relay/channel/openai/relay_responses.go @@ -350,6 +350,10 @@ func OaiResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp } }) + if apiErr := helper.FirstResultTimeoutError(info); apiErr != nil { + return nil, apiErr + } + if streamErr != nil { return nil, streamErr } diff --git a/relay/channel/openai/responses_via_chat.go b/relay/channel/openai/responses_via_chat.go index 6913c807..7d4c0655 100644 --- a/relay/channel/openai/responses_via_chat.go +++ b/relay/channel/openai/responses_via_chat.go @@ -122,6 +122,10 @@ func OaiChatToResponsesStreamHandler(c *gin.Context, info *relaycommon.RelayInfo } }) + if apiErr := helper.FirstResultTimeoutError(info); apiErr != nil { + return nil, apiErr + } + if streamErr != nil { return nil, streamErr } diff --git a/relay/channel/xai/text.go b/relay/channel/xai/text.go index 802611b7..bb7467f9 100644 --- a/relay/channel/xai/text.go +++ b/relay/channel/xai/text.go @@ -66,6 +66,9 @@ func xAIStreamHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.Re sr.Error(err) } }) + if apiErr := helper.FirstResultTimeoutError(info); apiErr != nil { + return nil, apiErr + } if !containStreamUsage { usage = service.ResponseText2Usage(c, responseTextBuilder.String(), info.UpstreamModelName, info.GetEstimatePromptTokens()) diff --git a/relay/common/channel_health_observation_test.go b/relay/common/channel_health_observation_test.go index 7b8515be..ee5f21de 100644 --- a/relay/common/channel_health_observation_test.go +++ b/relay/common/channel_health_observation_test.go @@ -14,6 +14,12 @@ import ( ) func TestRelayInfoRecordsOneFirstResponsePerChannelAttempt(t *testing.T) { + oldSetting := *operation_setting.GetMonitorSetting() + *operation_setting.GetMonitorSetting() = operation_setting.MonitorSetting{ + AutoPriorityDemotionEnabled: true, + } + t.Cleanup(func() { *operation_setting.GetMonitorSetting() = oldSetting }) + oldRecorder := recordChannelHealthObservation t.Cleanup(func() { recordChannelHealthObservation = oldRecorder }) @@ -42,6 +48,7 @@ func TestRelayInfoRecordsOneFirstResponsePerChannelAttempt(t *testing.T) { require.Equal(t, 936, observations[0].ChannelID) require.Equal(t, 2, observations[0].RetryIndex) require.True(t, observations[0].AttemptLatencyMS >= 0) + require.True(t, observations[0].PolicyEnabled) } func TestRelayInfoFirstResponseEvaluationHasBoundedGrace(t *testing.T) { diff --git a/relay/common/relay_info.go b/relay/common/relay_info.go index b0438ad7..e1d3e43e 100644 --- a/relay/common/relay_info.go +++ b/relay/common/relay_info.go @@ -881,6 +881,10 @@ func (info *RelayInfo) recordChannelHealthObservation(event string) { if info == nil || info.ChannelMeta == nil || info.ChannelId <= 0 { return } + setting := operation_setting.GetMonitorSetting() + if setting == nil || (!setting.AutoPriorityDemotionEnabled && !setting.TimeoutAutoDisableEnabled) { + return + } start := info.channelAttemptStartTime if start.IsZero() { start = info.StartTime @@ -893,6 +897,7 @@ func (info *RelayInfo) recordChannelHealthObservation(event string) { _ = recordChannelHealthObservation(common.ChannelHealthObservation{ ChannelID: info.ChannelId, Event: event, + PolicyEnabled: true, ObservedAt: observedAt, AttemptLatencyMS: latency, Stream: info.IsStream, diff --git a/relay/helper/first_result_timeout.go b/relay/helper/first_result_timeout.go new file mode 100644 index 00000000..d7d17a01 --- /dev/null +++ b/relay/helper/first_result_timeout.go @@ -0,0 +1,26 @@ +package helper + +import ( + "errors" + "net/http" + + relaycommon "github.com/MAX-API-Next/MAX-API/relay/common" + "github.com/MAX-API-Next/MAX-API/types" +) + +// FirstResultTimeoutError converts a stream scanner timeout that happened +// before any channel response was recorded into the retryable API error used +// by relay settlement and retry handling. A stream that already delivered a +// channel response keeps its existing terminal handling. +func FirstResultTimeoutError(info *relaycommon.RelayInfo) *types.MaxAPIError { + if info == nil || info.StreamStatus == nil || + info.StreamStatus.EndReason != relaycommon.StreamEndReasonTimeout || + info.HasRecordedChannelFirstResponse() { + return nil + } + return types.NewOpenAIError( + errors.New("upstream stream timed out before the first response"), + types.ErrorCodeChannelResponseTimeExceeded, + http.StatusRequestTimeout, + ) +} diff --git a/relay/helper/first_result_timeout_test.go b/relay/helper/first_result_timeout_test.go new file mode 100644 index 00000000..ec91fe77 --- /dev/null +++ b/relay/helper/first_result_timeout_test.go @@ -0,0 +1,30 @@ +package helper + +import ( + "testing" + + relaycommon "github.com/MAX-API-Next/MAX-API/relay/common" + "github.com/MAX-API-Next/MAX-API/types" + "github.com/stretchr/testify/require" +) + +func TestFirstResultTimeoutErrorRequiresMissingFirstResponse(t *testing.T) { + info := &relaycommon.RelayInfo{StreamStatus: relaycommon.NewStreamStatus()} + info.StreamStatus.SetEndReason(relaycommon.StreamEndReasonTimeout, nil) + + err := FirstResultTimeoutError(info) + require.NotNil(t, err) + require.Equal(t, types.ErrorCodeChannelResponseTimeExceeded, err.GetErrorCode()) + + info.SetFirstResponseTime() + require.Nil(t, FirstResultTimeoutError(info)) + + info.StreamStatus = relaycommon.NewStreamStatus() + info.StreamStatus.SetEndReason(relaycommon.StreamEndReasonDone, nil) + require.Nil(t, FirstResultTimeoutError(info)) +} + +func TestFirstResultTimeoutErrorHandlesNilState(t *testing.T) { + require.Nil(t, FirstResultTimeoutError(nil)) + require.Nil(t, FirstResultTimeoutError(&relaycommon.RelayInfo{})) +} diff --git a/service/http_client.go b/service/http_client.go index a04ad55a..ef93331b 100644 --- a/service/http_client.go +++ b/service/http_client.go @@ -20,6 +20,10 @@ var ( ssrfProtectedHTTPClient *http.Client proxyClientLock sync.Mutex proxyClients = make(map[string]*http.Client) + proxyClientResetHooks struct { + sync.RWMutex + hooks []func() + } ssrfProtectionCacheLock sync.RWMutex ssrfProtectionCacheKey string @@ -159,13 +163,33 @@ func GetHttpClientWithProxy(proxyURL string) (*http.Client, error) { // ResetProxyClientCache 清空代理客户端缓存，确保下次使用时重新初始化 func ResetProxyClientCache() { proxyClientLock.Lock() - defer proxyClientLock.Unlock() for _, client := range proxyClients { if transport, ok := client.Transport.(*http.Transport); ok && transport != nil { transport.CloseIdleConnections() } } proxyClients = make(map[string]*http.Client) + proxyClientLock.Unlock() + + proxyClientResetHooks.RLock() + hooks := append([]func(){}, proxyClientResetHooks.hooks...) + proxyClientResetHooks.RUnlock() + for _, hook := range hooks { + hook() + } +} + +// RegisterProxyClientCacheResetHook registers a package-local cache cleanup +// callback. The callback runs after proxy clients have been closed and removed. +// It lets adapters retire derived transport caches without introducing an +// import cycle back into the service package. +func RegisterProxyClientCacheResetHook(hook func()) { + if hook == nil { + return + } + proxyClientResetHooks.Lock() + proxyClientResetHooks.hooks = append(proxyClientResetHooks.hooks, hook) + proxyClientResetHooks.Unlock() } // NewProxyHttpClient 创建支持代理的 HTTP 客户端 @@ -177,17 +201,19 @@ func NewProxyHttpClient(proxyURL string) (*http.Client, error) { return http.DefaultClient, nil } - proxyClientLock.Lock() - if client, ok := proxyClients[proxyURL]; ok { - proxyClientLock.Unlock() - return client, nil - } - proxyClientLock.Unlock() - parsedURL, err := url.Parse(proxyURL) if err != nil { return nil, err } + if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" && parsedURL.Scheme != "socks5" && parsedURL.Scheme != "socks5h" { + return nil, fmt.Errorf("unsupported proxy scheme: %s, must be http, https, socks5 or socks5h", parsedURL.Scheme) + } + + proxyClientLock.Lock() + defer proxyClientLock.Unlock() + if client, ok := proxyClients[proxyURL]; ok { + return client, nil + } switch parsedURL.Scheme { case "http", "https": @@ -206,9 +232,7 @@ func NewProxyHttpClient(proxyURL string) (*http.Client, error) { CheckRedirect: checkRedirect, } client.Timeout = time.Duration(common.RelayTimeout) * time.Second - proxyClientLock.Lock() proxyClients[proxyURL] = client - proxyClientLock.Unlock() return client, nil case "socks5", "socks5h": @@ -246,9 +270,7 @@ func NewProxyHttpClient(proxyURL string) (*http.Client, error) { client := &http.Client{Transport: transport, CheckRedirect: checkRedirect} client.Timeout = time.Duration(common.RelayTimeout) * time.Second - proxyClientLock.Lock() proxyClients[proxyURL] = client - proxyClientLock.Unlock() return client, nil default: diff --git a/service/http_client_test.go b/service/http_client_test.go index 59c7a026..7ed951b4 100644 --- a/service/http_client_test.go +++ b/service/http_client_test.go @@ -1,9 +1,11 @@ package service import ( + "fmt" "net/http" "net/http/httptest" "strings" + "sync" "testing" "github.com/MAX-API-Next/MAX-API/common" @@ -11,6 +13,35 @@ import ( "github.com/stretchr/testify/require" ) +func TestNewProxyHttpClientSerializesConcurrentCreation(t *testing.T) { + ResetProxyClientCache() + t.Cleanup(ResetProxyClientCache) + + const proxyURL = "http://127.0.0.1:65531" + const callers = 16 + clients := make([]*http.Client, callers) + errs := make(chan error, callers) + var wg sync.WaitGroup + for i := 0; i < callers; i++ { + wg.Add(1) + go func(index int) { + defer wg.Done() + clients[index], _ = NewProxyHttpClient(proxyURL) + if clients[index] == nil { + errs <- fmt.Errorf("caller %d received nil client", index) + } + }(i) + } + wg.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } + for i := 1; i < callers; i++ { + require.Same(t, clients[0], clients[i]) + } +} + func TestNewSSRFProtectedHTTPClientDisablesEnvironmentProxy(t *testing.T) { t.Setenv("HTTP_PROXY", "http://127.0.0.1:8080") t.Setenv("HTTPS_PROXY", "http://127.0.0.1:8080") From 4ae54f65e5be48163c4a5bdb301b25b2d3e3e9b0 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Sat, 3 Oct 2026 10:31:27 +0800 Subject: [PATCH 12/12] fix: batch stale channel health cleanup --- pkg/channelhealth/runtime.go | 13 +++++++++++-- pkg/channelhealth/runtime_test.go | 5 ++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/pkg/channelhealth/runtime.go b/pkg/channelhealth/runtime.go index 51000baf..b936b3c0 100644 --- a/pkg/channelhealth/runtime.go +++ b/pkg/channelhealth/runtime.go @@ -398,6 +398,7 @@ func listRuntimeStatesByMembers(ctx context.Context, members []string) ([]Runtim return nil, err } states := make([]RuntimeState, 0, len(commands)) + staleMembers := make([]string, 0) for index, command := range commands { channelID, _ := strconv.Atoi(validMembers[index]) fields, err := command.Result() @@ -405,12 +406,20 @@ func listRuntimeStatesByMembers(ctx context.Context, members []string) ([]Runtim return nil, err } if len(fields) == 0 { - _ = maxcommon.RDB.Eval(ctx, removeStaleRuntimeMemberScript, - []string{runtimeStateKey(channelID), runtimeStateIndexKey}, validMembers[index]).Err() + staleMembers = append(staleMembers, validMembers[index]) continue } states = append(states, decodeRuntimeState(channelID, fields)) } + if len(staleMembers) > 0 { + cleanup := maxcommon.RDB.Pipeline() + for _, member := range staleMembers { + channelID, _ := strconv.Atoi(member) + cleanup.Eval(ctx, removeStaleRuntimeMemberScript, + []string{runtimeStateKey(channelID), runtimeStateIndexKey}, member) + } + _, _ = cleanup.Exec(ctx) + } return states, nil } diff --git a/pkg/channelhealth/runtime_test.go b/pkg/channelhealth/runtime_test.go index f55029b0..dec3cc5e 100644 --- a/pkg/channelhealth/runtime_test.go +++ b/pkg/channelhealth/runtime_test.go @@ -521,7 +521,7 @@ func TestListRuntimeStatesCompletesLegacyMigrationWithIndexedMembers(t *testing. }).Result() require.NoError(t, err) } - require.NoError(t, maxcommon.RDB.SAdd(context.Background(), runtimeStateIndexKey, "937").Err()) + require.NoError(t, maxcommon.RDB.SAdd(context.Background(), runtimeStateIndexKey, "937", "939").Err()) states, err := ListRuntimeStates(context.Background()) require.NoError(t, err) @@ -532,6 +532,9 @@ func TestListRuntimeStatesCompletesLegacyMigrationWithIndexedMembers(t *testing. indexed, err := maxcommon.RDB.SIsMember(context.Background(), runtimeStateIndexKey, "938").Result() require.NoError(t, err) require.True(t, indexed) + indexed, err = maxcommon.RDB.SIsMember(context.Background(), runtimeStateIndexKey, "939").Result() + require.NoError(t, err) + require.False(t, indexed) } func TestRecoverRuntimeStateRequiresConfiguredSuccessfulProbes(t *testing.T) {