From ef3c5ab826f32ccea462bbbf846eab124b1bfe39 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Thu, 20 Aug 2026 00:48:54 +0800 Subject: [PATCH 01/28] add features --- README_preview.md | 417 +++++++ controller/channel_performance.go | 98 ++ controller/model_performance.go | 81 ++ controller/perf_metrics.go | 21 +- middleware/distributor.go | 4 +- model/channel_performance.go | 238 ++++ model/channel_performance_test.go | 207 ++++ model/main.go | 8 +- model/model_performance.go | 147 +++ model/model_performance_test.go | 101 ++ model/perf_metric.go | 96 +- pkg/perf_metrics/flush.go | 141 ++- pkg/perf_metrics/metrics.go | 786 +++++++++++-- pkg/perf_metrics/metrics_test.go | 676 +++++++++++ pkg/perf_metrics/types.go | 149 ++- router/api-router.go | 8 + service/channel_performance.go | 312 +++++ service/channel_performance_test.go | 185 +++ service/channel_select.go | 16 +- service/channel_select_routing_test.go | 2 + service/model_performance.go | 312 +++++ service/model_performance_test.go | 176 +++ .../layout/config/smart-ops.config.test.ts | 45 + .../layout/config/smart-ops.config.ts | 73 ++ .../layout/lib/sidebar-view-registry.ts | 6 +- .../src/components/profile-dropdown.tsx | 6 +- .../src/features/performance-metrics/types.ts | 26 + .../components/model-details-chart-utils.ts | 92 ++ .../components/model-details-charts.tsx | 59 +- .../components/model-details-performance.tsx | 143 ++- .../model-details-uptime-sparkline.tsx | 19 +- web/default/src/features/smart-ops/api.ts | 88 ++ .../src/features/smart-ops/index.test.tsx | 576 ++++++++++ web/default/src/features/smart-ops/index.tsx | 1000 ++++++++++++++++ .../features/smart-ops/lib/channel-detail.ts | 30 + .../features/smart-ops/lib/filters.test.ts | 60 + .../src/features/smart-ops/lib/filters.ts | 83 ++ .../src/features/smart-ops/lib/format.test.ts | 33 + .../src/features/smart-ops/lib/format.ts | 33 + .../features/smart-ops/lib/model-sort.test.ts | 142 +++ .../src/features/smart-ops/lib/model-sort.ts | 55 + .../src/features/smart-ops/lib/sort.test.ts | 156 +++ .../src/features/smart-ops/lib/sort.ts | 66 ++ .../smart-ops/model-performance.test.tsx | 685 +++++++++++ .../features/smart-ops/model-performance.tsx | 1020 +++++++++++++++++ web/default/src/features/smart-ops/types.ts | 161 +++ .../system-settings/maintenance/config.ts | 1 + .../maintenance/sidebar-modules-section.tsx | 6 + web/default/src/hooks/use-sidebar-config.ts | 20 +- web/default/src/hooks/use-sidebar-data.ts | 21 +- web/default/src/i18n/locales/en.json | 93 +- web/default/src/i18n/locales/fr.json | 93 +- web/default/src/i18n/locales/ja.json | 93 +- web/default/src/i18n/locales/ru.json | 93 +- web/default/src/i18n/locales/vi.json | 93 +- web/default/src/i18n/locales/zh.json | 93 +- web/default/src/routeTree.gen.ts | 91 ++ .../smart-ops/channel-performance.tsx | 34 + .../smart-ops/model-performance.tsx | 34 + .../smart-ops/production-performance.tsx | 27 + .../_authenticated/smart-ops/system-info.tsx | 32 + .../_authenticated/system-info/index.tsx | 4 +- 62 files changed, 9396 insertions(+), 270 deletions(-) create mode 100644 README_preview.md create mode 100644 controller/channel_performance.go create mode 100644 controller/model_performance.go create mode 100644 model/channel_performance.go create mode 100644 model/channel_performance_test.go create mode 100644 model/model_performance.go create mode 100644 model/model_performance_test.go create mode 100644 pkg/perf_metrics/metrics_test.go create mode 100644 service/channel_performance.go create mode 100644 service/channel_performance_test.go create mode 100644 service/model_performance.go create mode 100644 service/model_performance_test.go create mode 100644 web/default/src/components/layout/config/smart-ops.config.test.ts create mode 100644 web/default/src/components/layout/config/smart-ops.config.ts create mode 100644 web/default/src/features/pricing/components/model-details-chart-utils.ts create mode 100644 web/default/src/features/smart-ops/api.ts create mode 100644 web/default/src/features/smart-ops/index.test.tsx create mode 100644 web/default/src/features/smart-ops/index.tsx create mode 100644 web/default/src/features/smart-ops/lib/channel-detail.ts create mode 100644 web/default/src/features/smart-ops/lib/filters.test.ts create mode 100644 web/default/src/features/smart-ops/lib/filters.ts create mode 100644 web/default/src/features/smart-ops/lib/format.test.ts create mode 100644 web/default/src/features/smart-ops/lib/format.ts create mode 100644 web/default/src/features/smart-ops/lib/model-sort.test.ts create mode 100644 web/default/src/features/smart-ops/lib/model-sort.ts create mode 100644 web/default/src/features/smart-ops/lib/sort.test.ts create mode 100644 web/default/src/features/smart-ops/lib/sort.ts create mode 100644 web/default/src/features/smart-ops/model-performance.test.tsx create mode 100644 web/default/src/features/smart-ops/model-performance.tsx create mode 100644 web/default/src/features/smart-ops/types.ts create mode 100644 web/default/src/routes/_authenticated/smart-ops/channel-performance.tsx create mode 100644 web/default/src/routes/_authenticated/smart-ops/model-performance.tsx create mode 100644 web/default/src/routes/_authenticated/smart-ops/production-performance.tsx create mode 100644 web/default/src/routes/_authenticated/smart-ops/system-info.tsx diff --git a/README_preview.md b/README_preview.md new file mode 100644 index 00000000..bb7c8fa2 --- /dev/null +++ b/README_preview.md @@ -0,0 +1,417 @@ +
+ +![MAX API](./web/default/public/logo.png) + +# MAX API + +**AI Models 与 Agents 的统一接入、治理和运营基础设施** + +把多供应商模型、Agent 工作负载、成本、路由、日志和权限放进同一套可控边界。 + +

+ 简体中文 | + 繁體中文 | + English | + Français | + 日本語 +

+ +

+ + license + + release + + docker + + GoReportCard + +

+ +

+ 快速开始 • + 核心能力 • + 生产部署 • + 常见问题 • + 资源与社区 +

+ +
+ +--- + +MAX API 是由来自科研机构和高校的 AGI 爱好者组织发起、维护和长期运营的开源项目。它位于应用、Agent 与上游模型服务之间，提供统一接口、身份鉴权、渠道路由、计费结算、日志观测和组织治理能力。 + +它不替代 Dify、LangChain、MCP Server 或业务 Agent，也不提供上游模型账号和 API Key。MAX API 的职责，是帮助已有 AI 应用和 Agent 更稳定、更可控地进入真实运行环境。 + +> [!IMPORTANT] +> 面向公众提供生成式人工智能服务时，使用者应自行完成所在司法辖区要求的上游授权、备案许可、内容安全、实名、日志留存、税务、支付和用户协议等合规义务。日志审计和内容留存仅应在具备合法依据、明确告知、权限隔离和数据安全措施的场景下启用。 + +> [!TIP] +> 生产环境建议使用正式版。Preview 版本用于测试和灰度验证，升级前请备份数据库并准备回滚方案。 + +## 🚀 快速开始 + +本地体验默认使用 SQLite，只需 Docker： + +```bash +docker pull cscitechtop/max-api:latest + +docker run --name max-api -d --restart always -p 3000:3000 -e TZ=Asia/Shanghai -v ./data:/data cscitechtop/max-api:latest +``` + +启动后访问： + +接下来只需完成三件事： + +1. 创建或确认管理员账号。 +2. 添加一个具有合法授权的上游渠道和 API Key。 +3. 创建访问令牌，在应用中把 Base URL 指向 MAX API。 + +> [!WARNING] +> SQLite 适合本地体验、开发和小规模测试。正式环境建议使用 MySQL ≥ 5.7.8 或 PostgreSQL ≥ 9.6，并配置 Redis、HTTPS、备份和恢复方案。 + +## ✨ 核心能力 + +| 能力 | MAX API 解决的问题 | +|---|---| +| 统一模型入口 | 用统一网关接入 OpenAI Compatible、Responses、Claude Messages、Gemini、Realtime 和多模态任务接口 | +| 多供应商路由 | 管理渠道、权重、分组、模型映射、失败重试和跨供应商切换，降低单一上游风险 | +| Agent 治理 / AgentOps | 为 Agent、工作流和工具调用分配独立令牌、模型范围、额度与调用记录 | +| 成本与计费 | 支持倍率、固定价格、表达式计费、异步任务 rate-card、预扣费、结算和失败退款 | +| 可观测与审计 | 按用户、令牌、模型、渠道、分组和节点查看用量、耗时、错误、重试与管理员审计信息 | +| 私有化与扩展 | 支持 SQLite、MySQL、PostgreSQL、Redis、多节点部署和自定义上游协议配置 | + +### 使用 MAX API 的优势 + +| 维度 | 直接连接多个供应商 | 使用 MAX API | +|---|---|---| +| 应用接入 | 每家 SDK、协议和错误格式分别维护 | 应用侧使用相对稳定的统一入口 | +| 模型切换 | 需要修改代码和配置 | 通过渠道、映射、分组和路由调整 | +| 成本核算 | 账单分散，难以归因 | 按用户、令牌、模型、渠道和节点统一统计 | +| 权限治理 | 凭据和权限散落在不同应用 | 集中管理令牌、模型范围、额度和有效期 | +| 故障排查 | 日志分散，跨供应商关联困难 | 在网关侧统一查看请求、重试、耗时和错误 | + +## 🎯 适用场景 + +- **团队或组织内部模型网关**：统一管理用户、令牌、模型、供应商、权限和费用。 +- **Agent 应用运行底座**：为 Agent、工作流和工具调用提供模型访问控制、成本归因和异常定位。 +- **多供应商容灾与迁移**：在不同模型平台之间进行映射、加权路由、失败重试和灰度切换。 +- **多模态任务治理**：统一管理图像、音频、视频、嵌入、重排序和实时对话接口。 +- **私有化与合规运营**：自主管理密钥、数据、日志、审计、价格和部署环境。 + +## 🧭 工作方式 + +![MAX API 系统架构图](./docs/images/MAX-API架构图.png) + +一次模型请求通常经过以下链路： + +```text +应用 / SDK / Agent + → 统一接口与身份鉴权 + → 模型权限、限流和安全检查 + → 渠道选择、映射与失败重试 + → 上游协议适配 + → 用量结算、日志和审计 +``` + +MAX API 保持分层架构：`Router → Controller → Service → Model`，供应商协议转换位于 `relay/` 与 `relay/channel/`。数据层使用 GORM，兼容 SQLite、MySQL 和 PostgreSQL；Redis 与内存缓存用于限流、会话和多节点协同。 + +
+开发者技术栈 + +| 层 | 技术 | +|---|---| +| 后端 | Go 1.25.1+、Gin、GORM v2 | +| 前端 | React 19、TypeScript、Rsbuild、Base UI、Tailwind CSS | +| 前端包管理 | Bun workspace | +| 数据库 | SQLite / MySQL ≥ 5.7.8 / PostgreSQL ≥ 9.6 | +| 缓存 | Redis + 内存缓存 | +| 鉴权 | JWT、WebAuthn/Passkeys、OAuth、OIDC | + +
+ +## 🤖 模型与接口 + +> 实际可用能力取决于你的上游授权、渠道配置、模型映射和供应商支持。MAX API 负责治理这些能力，不提供模型服务本身。 + +### 主要接口 + +| 类别 | 接口或能力 | +|---|---| +| Chat / Responses | `/v1/chat/completions`、`/v1/responses`，以及 Responses ↔ Chat Completions 兼容转换 | +| Embeddings / Rerank | `/v1/embeddings`、`/v1/rerank` | +| Images / Audio / Video | `/v1/images/*`、`/v1/audio/*`、`/v1/videos/*` | +| 原生协议 | Claude Messages、Google Gemini、OpenAI Realtime 等入口 | +| 异步任务 | 任务提交、轮询、状态映射、结果代理和参数化计费 | +| 自定义上游 | Base URL、路径覆盖、参数覆盖、Header 覆盖、状态和结果字段映射 | + +### 供应商与生态 + +可接入 OpenAI、Azure OpenAI、Claude、Gemini、AWS Bedrock、Vertex AI、Ollama，以及 DeepSeek、通义千问 / 阿里云百炼、智谱 GLM、Kimi、豆包 / 火山引擎、腾讯混元、百度文心 / 千帆、讯飞星火、MiniMax、零一万物、硅基流动等平台。 + +同时支持 Codex、Dify、RAGFlow、Kling、Seedance、Midjourney、Suno 等应用、Agent 或多模态服务相关的接入治理。具体能力以渠道类型和当前版本为准。 + +
+Reasoning Effort 模型命名示例 + +- OpenAI：`o3-mini-high`、`gpt-5-medium`、`gpt-5-low` +- Claude：`claude-3-7-sonnet-20250219-thinking` +- Gemini：`gemini-2.5-flash-thinking`、`gemini-2.5-pro-thinking-128` + +
+ +## 🛡️ 治理与运营 + +推荐按照下面的顺序配置： + +1. 配置登录、安全限制和用户注册策略。 +2. 添加上游渠道，确认 Base URL、模型列表、能力和合法授权凭据。 +3. 按团队或业务线设置用户分组、令牌分组和模型范围。 +4. 配置模型价格、任务计费、额度、失败重试和日志策略。 +5. 为每个应用、Agent 或环境创建独立令牌。 +6. 通过使用日志、数据看板和渠道测试持续观察运行状态。 + +| 管理入口 | 用途 | +|---|---| +| 渠道管理 | 维护供应商、模型映射、权重、协议路径、能力矩阵和配置校验 | +| 模型与价格 | 维护模型列表、倍率、固定价格、表达式计费和任务 rate-card | +| 令牌管理 | 为应用、Agent、工作流和用户限制模型范围、额度和有效期 | +| 使用日志 | 查看消耗、耗时、错误、重试、渠道命中和审计信息 | +| 系统设置 | 管理安全、性能、日志、运营、计费、支付和站点配置 | +| 数据看板 | 查看请求量、模型用量、消费趋势和渠道运行状态 | + +
+高级治理能力 + +- **渠道能力矩阵与校验**：识别 `chat/completions`、`responses`、`embeddings`、`rerank`、视频任务和模型发现能力，并提示 Base URL、JSON、Vertex AI、Codex 凭据及任务路径配置风险。 +- **通用视频任务协议**：可配置提交路径、查询路径、任务 ID、状态、进度、结果 URL 和错误字段映射；请求体透传使用 `Pass Through Body`，字段改写使用 `Param Override`。 +- **计费 JSON 维护**：支持批量维护表达式计费和任务 rate-card；视频任务可按时长、清晰度、音频等请求参数匹配价格。 +- **管理员日志审计**：普通用户日志接口会过滤管理员专用字段；启用内容留存前应确认合规依据和访问权限。 +- **性能治理**：支持模型限流、流式超时、大响应缓冲、请求体限制、磁盘缓存、Pyroscope 和优雅关闭。 + +完整配置参考请查看 [详细中文文档](./README.zh_CN.md)。 + +
+ +## 🚢 生产部署 + +推荐使用 Docker Compose： + +```bash +git clone https://github.com/MAX-API-Next/MAX-API.git +cd MAX-API + +# 修改 docker-compose.yml 中的数据库、Redis 密码和密钥 +docker compose up -d +``` + +### 部署要求 + +| 组件 | 建议 | +|---|---| +| 数据库 | MySQL ≥ 5.7.8 或 PostgreSQL ≥ 9.6，并配置备份与恢复 | +| 缓存 | 单机可使用内存缓存，多机部署建议使用 Redis | +| 入口 | HTTPS 反向代理、请求大小限制和可信网络策略 | +| 密钥 | 显式配置随机 `SESSION_SECRET`；Redis/多节点场景配置统一 `CRYPTO_SECRET` | +| 节点 | 多节点设置稳定且唯一的 `NODE_NAME` | +| 日志 | 根据合规和运维需要配置日志目录、LOG_DB、清理与保留策略 | + +
+常用环境变量 + +| 变量 | 用途 | 默认值 | +|---|---|---| +| `SQL_DSN` | 主数据库连接 | SQLite | +| `LOG_SQL_DSN` | 可选独立日志数据库 | 与主数据库相同 | +| `REDIS_CONN_STRING` | Redis 连接 | - | +| `SESSION_SECRET` | 会话密钥，多机必须一致 | - | +| `CRYPTO_SECRET` | 加密密钥，Redis/多机必须一致 | - | +| `NODE_NAME` | 节点名称与日志归属 | - | +| `STREAMING_TIMEOUT` | 流式响应超时，单位秒 | `300` | +| `STREAM_SCANNER_MAX_BUFFER_MB` | 流式单行最大缓冲 | `64` | +| `MAX_REQUEST_BODY_MB` | 解压后请求体上限 | `32` | +| `ERROR_LOG_ENABLED` | 错误请求日志开关 | `false` | +| `PYROSCOPE_URL` | Pyroscope 服务地址 | - | + +
+ +
+多节点部署注意事项 + +- 所有节点必须使用相同的 `SESSION_SECRET`。 +- 使用共享 Redis 时必须使用相同的 `CRYPTO_SECRET`。 +- 每个节点应设置不同且稳定的 `NODE_NAME`。 +- 使用外部数据库、外部 Redis、统一 HTTPS 入口和可靠备份。 + +
+ +
+从源码构建镜像 + +```bash +git clone https://github.com/MAX-API-Next/MAX-API.git +cd MAX-API + +go mod download +go mod verify +docker build --pull --no-cache -t cscitechtop/max-api:latest . +``` + +前端使用 Bun workspace。构建上下文需要保留 `web/package.json`、`web/bun.lock` 和 `web/default/package.json`，否则 `catalog:` 依赖可能无法解析。 + +
+ +## 🗺️ 路线图 + +以下方向会根据维护节奏和真实场景调整，不构成发布日期承诺： + +- 深化模型目录、权限、能力标签、价格和供应商变化治理。 +- 完善 Agent 调用追踪、成本归因、异常定位、Evidence 与 AgentOps。 +- 以受控插件形式提供 Detector、OpsAgent、评估器和行业治理能力。 +- 增强多模态任务、协议转换、国产模型和供应商模板。 +- 完善组织级权限、审计、私有化、多节点和运营体验。 + +欢迎通过 [GitHub Issues](https://github.com/MAX-API-Next/MAX-API/issues) 提交需求和改进建议。 + +## ❓ 常见问题 + +
+MAX API 会提供模型服务或 API Key 吗？ + +不会。MAX API 是模型与 Agent 工作负载的网关治理层。你需要自行获得合法授权的上游模型服务和凭据。 + +
+ +
+MAX API 和 Agent 框架是什么关系？ + +MAX API 不替代 Dify、LangChain、MCP Server、工作流引擎或业务 Agent。它负责这些应用与上游模型之间的访问、路由、成本、日志和治理。 + +
+ +
+支持哪些数据库？ + +支持 SQLite、MySQL ≥ 5.7.8 和 PostgreSQL ≥ 9.6。SQLite 适合本地体验；生产建议使用 MySQL 或 PostgreSQL。 + +
+ +
+能否从 One API / New API 迁移？ + +项目兼容 One API 与 New API 的主要数据结构，但迁移前仍应备份数据库，并在测试环境验证渠道、倍率、用户、令牌、任务和日志数据。 + +
+ +
+图像、流式响应或大响应被截断怎么办？ + +可以调整 `STREAM_SCANNER_MAX_BUFFER_MB`。请求体返回 `413` 时检查 `MAX_REQUEST_BODY_MB`。提高限制前请同时评估内存与并发风险。 + +
+ +
+多机部署登录状态不一致怎么办？ + +确认所有节点使用相同的 `SESSION_SECRET`。使用共享 Redis 时，还必须统一 `CRYPTO_SECRET`，并为节点设置稳定且不同的 `NODE_NAME`。 + +
+ +
+普通用户能看到管理员日志审计内容吗？ + +普通用户日志接口会过滤管理员专用字段。数据库管理员、系统管理员或拥有管理员日志权限的人仍可能访问相关数据，因此必须严格控制权限和留存策略。 + +
+ +## 🔗 资源与社区 + +| 资源 | 链接 | +|---|---| +| 项目仓库 | [MAX-API-Next/MAX-API](https://github.com/MAX-API-Next/MAX-API) | +| 问题反馈 | [GitHub Issues](https://github.com/MAX-API-Next/MAX-API/issues) | +| 最新发布 | [GitHub Releases](https://github.com/MAX-API-Next/MAX-API/releases) | +| Docker 镜像 | [cscitechtop/max-api](https://hub.docker.com/r/cscitechtop/max-api) | +| DeepWiki | [Ask DeepWiki](https://deepwiki.com/MAX-API-Next/MAX-API) | +| 社群交流 | QQ 群 `950126533`；微信群搜索 `MAX-API` | + +配套工具： + +- [max-api-key-tool](https://github.com/MAX-API-Next/MAX-API-key-tool)：Key 额度查询工具。 +- [max-api-horizon](https://github.com/MAX-API-Next/MAX-API-horizon)：MAX API 高性能优化版。 + +## 🤝 项目来源与二次开发 + +MAX API 基于以下开源项目持续增强 AI API 网关与治理能力、扩展功能并修复问题： + +| 项目 | 许可证 | +|---|---| +| [One API](https://github.com/songquanpeng/one-api) | MIT | +| [New API](https://github.com/QuantumNous/new-api) | AGPLv3 | +| [Midjourney-Proxy](https://github.com/novicezk/midjourney-proxy) | Apache-2.0 | +| [Suno API](https://github.com/Suno-API/Suno-API) | MIT | + +如果你基于本项目进行二次开发并仅供自用，欢迎在项目主页、页脚或“关于”页面等明显位置，任选一种方式保留项目来源或社区鸣谢： + +- 添加项目地址：[MAX-API-Next/MAX-API](https://github.com/MAX-API-Next/MAX-API) +- 鸣谢社区：[MAX-API-Next](https://github.com/MAX-API-Next) + +
+React / Tailwind CSS 鸣谢代码示例 + +```tsx +

+ 基于{' '} + + MAX-API-Next/MAX-API + {' '} + 二次开发 · 感谢{' '} + + MAX-API-Next 社区 + +

+``` + +
+ +满足上述任一展示要求并保持链接清晰可见，即自动获得本项目的临时商用授权，无需另行申请或等待确认。该授权为非永久授权，仅在持续满足展示要求期间有效；有效期及后续调整以项目方在本 README 或官方社区发布的最新说明为准。 + +商业使用本项目时，还须分别遵守 One API 的 MIT 许可证与 New API 的 AGPLv3 许可证，具体以各上游项目的 `LICENSE` 文件为准。本项目提供的临时商用授权不替代、也不免除相关上游项目的开源许可义务。 + +如不再满足展示要求，或临时授权到期、被公告调整或终止，仍需按 AGPLv3 或项目方另行书面授权使用本项目。如需长期商用授权，请联系：maxapi@max-api.ai。 + +## 📜 许可证 + +本项目采用 [GNU Affero 通用公共许可证 v3.0（AGPLv3）](./LICENSE) 授权。 + +除默认的 AGPLv3 授权外，符合上文“项目来源与二次开发”条件的自用二开项目，可按该说明自动获得非永久的临时商用授权。该临时授权仅覆盖 MAX API 项目方有权授权的新增与修改部分，并不包含或代替 One API、New API 等上游项目的许可授权。 + +如果你修改并通过网络向用户提供本项目服务，请理解并遵守 AGPLv3 对应源码提供等义务。商业合作、机构合作或其他授权问题，请联系：maxapi@max-api.ai。 + +--- + +
+ +### 💖 感谢使用 MAX API + +如果这个项目对你有帮助，欢迎给我们一个 ⭐ Star。 + +**[官方文档](https://github.com/MAX-API-Next/MAX-API)** • **[问题反馈](https://github.com/MAX-API-Next/MAX-API/issues)** • **[最新发布](https://github.com/MAX-API-Next/MAX-API/releases)** + +Built with ❤️ by MAX-API-Next + +
diff --git a/controller/channel_performance.go b/controller/channel_performance.go new file mode 100644 index 00000000..d48201d0 --- /dev/null +++ b/controller/channel_performance.go @@ -0,0 +1,98 @@ +package controller + +import ( + "errors" + "net/http" + "strconv" + "strings" + + "github.com/MAX-API-Next/MAX-API/logger" + "github.com/MAX-API-Next/MAX-API/service" + "github.com/gin-gonic/gin" +) + +// GetChannelPerformance exposes the read-only legacy-log projection used by +// the administrator Smart Operations Center. It never probes, reroutes, +// disables or mutates a channel. +func GetChannelPerformance(c *gin.Context) { + query := service.ChannelPerformanceQuery{ + StartAt: parseInt64Query(c, "start"), + EndAt: parseInt64Query(c, "end"), + Hours: parseIntQuery(c, "hours"), + Limit: parseIntQuery(c, "limit"), + ChannelID: parseIntQuery(c, "channel_id"), + ModelName: strings.TrimSpace(c.Query("model")), + Group: strings.TrimSpace(c.Query("group")), + } + + result, err := service.GetChannelPerformance(c.Request.Context(), query) + if err != nil { + status := http.StatusInternalServerError + if errors.Is(err, service.ErrInvalidChannelPerformanceQuery) { + status = http.StatusBadRequest + } else { + logger.LogError(c.Request.Context(), "failed to query channel performance: "+err.Error()) + } + message := err.Error() + if status == http.StatusInternalServerError { + message = "failed to query channel performance" + } + c.JSON(status, gin.H{ + "success": false, + "message": message, + }) + return + } + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": result, + }) +} + +// GetChannelPerformanceDetail exposes a channel-scoped 24-hour projection. +// It is queried only when an administrator opens the channel detail sheet. +func GetChannelPerformanceDetail(c *gin.Context) { + result, err := service.GetChannelPerformanceDetail( + c.Request.Context(), + parseIntQuery(c, "channel_id"), + ) + if err != nil { + status := http.StatusInternalServerError + if errors.Is(err, service.ErrInvalidChannelPerformanceQuery) { + status = http.StatusBadRequest + } else { + logger.LogError(c.Request.Context(), "failed to query channel performance detail: "+err.Error()) + } + message := err.Error() + if status == http.StatusInternalServerError { + message = "failed to query channel performance detail" + } + c.JSON(status, gin.H{ + "success": false, + "message": message, + }) + return + } + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": result, + }) +} + +func parseIntQuery(c *gin.Context, key string) int { + value, err := strconv.Atoi(c.Query(key)) + if err != nil { + return 0 + } + return value +} + +func parseInt64Query(c *gin.Context, key string) int64 { + value, err := strconv.ParseInt(c.Query(key), 10, 64) + if err != nil { + return 0 + } + return value +} diff --git a/controller/model_performance.go b/controller/model_performance.go new file mode 100644 index 00000000..2a7f0ee2 --- /dev/null +++ b/controller/model_performance.go @@ -0,0 +1,81 @@ +package controller + +import ( + "errors" + "net/http" + "strings" + + "github.com/MAX-API-Next/MAX-API/logger" + "github.com/MAX-API-Next/MAX-API/service" + "github.com/gin-gonic/gin" +) + +// GetModelPerformance exposes the read-only legacy-log projection used by the +// administrator Smart Operations Center. It aggregates eligible production +// logs by model and never mutates, probes or reroutes a channel. +func GetModelPerformance(c *gin.Context) { + query := service.ModelPerformanceQuery{ + StartAt: parseInt64Query(c, "start"), + EndAt: parseInt64Query(c, "end"), + Hours: parseIntQuery(c, "hours"), + Limit: parseIntQuery(c, "limit"), + ModelName: strings.TrimSpace(c.Query("model")), + Group: strings.TrimSpace(c.Query("group")), + } + + result, err := service.GetModelPerformance(c.Request.Context(), query) + if err != nil { + status := http.StatusInternalServerError + if errors.Is(err, service.ErrInvalidModelPerformanceQuery) { + status = http.StatusBadRequest + } else { + logger.LogError(c.Request.Context(), "failed to query model performance: "+err.Error()) + } + message := err.Error() + if status == http.StatusInternalServerError { + message = "failed to query model performance" + } + c.JSON(status, gin.H{ + "success": false, + "message": message, + }) + return + } + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": result, + }) +} + +// GetModelPerformanceDetail exposes the existing perf_metrics projection to +// administrators without coupling Smart Operations to public pricing-module +// visibility. The detail window is fixed at the latest 24 hours. +func GetModelPerformanceDetail(c *gin.Context) { + result, err := service.GetModelPerformanceDetail( + c.Request.Context(), + strings.TrimSpace(c.Query("model")), + ) + if err != nil { + status := http.StatusInternalServerError + if errors.Is(err, service.ErrInvalidModelPerformanceQuery) { + status = http.StatusBadRequest + } else { + logger.LogError(c.Request.Context(), "failed to query model performance detail: "+err.Error()) + } + message := err.Error() + if status == http.StatusInternalServerError { + message = "failed to query model performance detail" + } + c.JSON(status, gin.H{ + "success": false, + "message": message, + }) + return + } + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": result, + }) +} diff --git a/controller/perf_metrics.go b/controller/perf_metrics.go index 1873c57e..6aa7ef44 100644 --- a/controller/perf_metrics.go +++ b/controller/perf_metrics.go @@ -20,11 +20,7 @@ func GetPerfMetricsSummary(c *gin.Context) { } } - activeGroups := lo.Keys(ratio_setting.GetGroupRatioCopy()) - for _, route := range setting.GetAutoRoutes() { - activeGroups = append(activeGroups, route.Key) - } - result, err := perfmetrics.QuerySummaryAll(hours, activeGroups) + result, err := perfmetrics.QuerySummaryAll(hours, activePerformanceGroups()) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{ "success": false, @@ -57,9 +53,10 @@ func GetPerfMetrics(c *gin.Context) { } result, err := perfmetrics.Query(perfmetrics.QueryParams{ - Model: modelName, - Group: c.Query("group"), - Hours: hours, + Model: modelName, + Group: c.Query("group"), + Hours: hours, + AllowedGroups: activePerformanceGroups(), }) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{ @@ -77,6 +74,14 @@ func GetPerfMetrics(c *gin.Context) { }) } +func activePerformanceGroups() []string { + groups := lo.Keys(ratio_setting.GetGroupRatioCopy()) + for _, route := range setting.GetAutoRoutes() { + groups = append(groups, route.Key) + } + return groups +} + func filterActiveGroups(groups []perfmetrics.GroupResult) []perfmetrics.GroupResult { activeRatios := ratio_setting.GetGroupRatioCopy() return lo.Filter(groups, func(g perfmetrics.GroupResult, _ int) bool { diff --git a/middleware/distributor.go b/middleware/distributor.go index 0b756482..d99f15bb 100644 --- a/middleware/distributor.go +++ b/middleware/distributor.go @@ -123,7 +123,7 @@ func Distribute() func(c *gin.Context) { channelSupportsRequestPath(preferred, c.Request.URL.Path) { if group, groupIndex, matched := findAffinityRouteGroup(routePlan, modelRequest.Model, preferred.Id, model.IsChannelEnabledForGroupModel); matched { selectGroup = group - common.SetContextKey(c, constant.ContextKeyAutoGroup, group) + service.SetSelectedRoutingGroup(c, group) channel = preferred affinityUsable = true service.MarkChannelAffinityUsed(c, group, preferred.Id) @@ -140,7 +140,7 @@ func Distribute() func(c *gin.Context) { for _, g := range autoGroups { if model.IsChannelEnabledForGroupModel(g, modelRequest.Model, preferred.Id) { selectGroup = g - common.SetContextKey(c, constant.ContextKeyAutoGroup, g) + service.SetSelectedRoutingGroup(c, g) channel = preferred affinityUsable = true service.MarkChannelAffinityUsed(c, g, preferred.Id) diff --git a/model/channel_performance.go b/model/channel_performance.go new file mode 100644 index 00000000..cd99cc8d --- /dev/null +++ b/model/channel_performance.go @@ -0,0 +1,238 @@ +package model + +import ( + "context" + "errors" + "fmt" + + "gorm.io/gorm" +) + +const ( + LegacyRetryMetricStateAvailable = "available" + LegacyRetryMetricStatePending = "pending" + LegacyRetryMetricStateUnavailable = "unavailable" +) + +const ( + legacyViolationFeePattern = `%"violation_fee":true%` + legacyTaskAdjustmentPattern = `%"pre_consumed_quota":%` + legacyTaskSubmissionPattern = `%"is_task":true%` + legacyViolationFeeContent = "Violation fee charged" +) + +// ChannelPerformanceQuery describes the bounded, read-only legacy log query +// used by Smart Operations Center. It deliberately does not model a Relay +// Attempt; a future Attempt reader can implement the service seam separately. +type ChannelPerformanceQuery struct { + StartAt int64 + EndAt int64 + Hours int + Limit int + ChannelID int + ModelName string + Group string +} + +// LegacyChannelPerformanceRow is an aggregate projection from LOG_DB. Keep +// this projection narrow so channel credentials, prompts and responses never +// enter the operations surface. +type LegacyChannelPerformanceRow struct { + ChannelID int `gorm:"column:channel_id"` + ModelName string `gorm:"column:model_name"` + EffectiveGroup string `gorm:"column:effective_group"` + ObservedCount int64 `gorm:"column:observed_count"` + ConsumeLogCount int64 `gorm:"column:consume_log_count"` + ErrorLogCount int64 `gorm:"column:error_log_count"` + ConsumedQuota int64 `gorm:"column:consumed_quota"` + RetryLogCount int64 `gorm:"column:retry_log_count"` + LatencySampleCount int64 `gorm:"column:latency_sample_count"` + TotalUseTimeSeconds int64 `gorm:"column:total_use_time_seconds"` + LastObservedAt int64 `gorm:"column:last_observed_at"` +} + +type LegacyChannelPerformanceSummary struct { + ChannelCount int64 `gorm:"column:channel_count"` + ObservedCount int64 `gorm:"column:observed_count"` + ConsumeLogCount int64 `gorm:"column:consume_log_count"` + ErrorLogCount int64 `gorm:"column:error_log_count"` + ConsumedQuota int64 `gorm:"column:consumed_quota"` + RetryLogCount int64 `gorm:"column:retry_log_count"` + LatencySampleCount int64 `gorm:"column:latency_sample_count"` + TotalUseTimeSeconds int64 `gorm:"column:total_use_time_seconds"` + LastObservedAt int64 `gorm:"column:last_observed_at"` +} + +type LegacyChannelPerformanceResult struct { + Rows []LegacyChannelPerformanceRow + Summary LegacyChannelPerformanceSummary + RetryMetricsAvailable bool + RetryMetricState string +} + +// ChannelPerformanceChannel is the safe metadata projection joined in the +// application after LOG_DB aggregation. It intentionally excludes Key, +// BaseURL, overrides, settings and all other secret-bearing fields. +type ChannelPerformanceChannel struct { + ID int `gorm:"column:id"` + Name string `gorm:"column:name"` + Type int `gorm:"column:type"` + Status int `gorm:"column:status"` + ResponseTime int `gorm:"column:response_time"` + TestTime int64 `gorm:"column:test_time"` +} + +// QueryLegacyChannelPerformance reads only existing log rows. The bounded row +// query returns one extra row for truncation detection, while the independent +// summary query always covers every matching eligible log. +func QueryLegacyChannelPerformance(ctx context.Context, query ChannelPerformanceQuery) (LegacyChannelPerformanceResult, error) { + if LOG_DB == nil { + return LegacyChannelPerformanceResult{}, fmt.Errorf("log database is not initialized") + } + + limit := query.Limit + if limit <= 0 { + limit = 100 + } + if limit > 200 { + limit = 200 + } + + // group is reserved by all supported databases. logGroupCol is selected by + // initCol for the configured LOG_DB dialect. + groupColumn := logGroupCol + if groupColumn == "" { + groupColumn = commonGroupCol + } + + retryMetricState := legacyChannelPerformanceRetryMetricState() + retryMetricsAvailable := retryMetricState == LegacyRetryMetricStateAvailable + rowSelect, rowArgs := legacyChannelPerformanceSelect(groupColumn, true, retryMetricsAvailable) + + var rows []LegacyChannelPerformanceRow + err := legacyChannelPerformanceQuery(ctx, query, groupColumn). + Select(rowSelect, rowArgs...). + Group("channel_id, model_name, " + groupColumn). + Order("observed_count DESC, last_observed_at DESC"). + Limit(limit + 1). + Scan(&rows).Error + if err != nil { + return LegacyChannelPerformanceResult{}, err + } + + summarySelect, summaryArgs := legacyChannelPerformanceSelect(groupColumn, false, retryMetricsAvailable) + var summary LegacyChannelPerformanceSummary + if err := legacyChannelPerformanceQuery(ctx, query, groupColumn). + Select(summarySelect, summaryArgs...). + Scan(&summary).Error; err != nil { + return LegacyChannelPerformanceResult{}, err + } + + return LegacyChannelPerformanceResult{ + Rows: rows, + Summary: summary, + RetryMetricsAvailable: retryMetricsAvailable, + RetryMetricState: retryMetricState, + }, nil +} + +func legacyChannelPerformanceRetryMetricState() string { + if err := ensureLogRetryMarkerBackfillCompletedForRead(); err == nil { + return LegacyRetryMetricStateAvailable + } else if errors.Is(err, ErrLogRetryMarkerBackfillIncomplete) { + return LegacyRetryMetricStatePending + } + return LegacyRetryMetricStateUnavailable +} + +func legacyChannelPerformanceQuery(ctx context.Context, query ChannelPerformanceQuery, groupColumn string) *gorm.DB { + tx := LOG_DB.WithContext(ctx).Table("logs") + tx = tx.Where("created_at >= ? AND created_at <= ?", query.StartAt, query.EndAt) + tx = tx.Where("type IN ?", []int{LogTypeConsume, LogTypeError}) + tx = tx.Where("channel_id > ?", 0).Where("model_name <> ?", "") + // Channel tests are probes, not production traffic. Billing-only consume + // records are accounting evidence, not successful Relay attempts. Keep the + // accounting rows intact and exclude them only from this read projection. + tx = tx.Where("(content IS NULL OR content <> ?)", "模型测试"). + Where("(token_name IS NULL OR token_name <> ?)", "模型测试"). + Where( + "(type <> ? OR ((other IS NULL OR (other NOT LIKE ? AND other NOT LIKE ?)) AND (content IS NULL OR content <> ?)))", + LogTypeConsume, + legacyViolationFeePattern, + legacyTaskAdjustmentPattern, + legacyViolationFeeContent, + ) + if query.ChannelID > 0 { + tx = tx.Where("channel_id = ?", query.ChannelID) + } + if query.ModelName != "" { + tx = tx.Where("model_name = ?", query.ModelName) + } + if query.Group != "" { + tx = tx.Where(groupColumn+" = ?", query.Group) + } + return tx +} + +func legacyChannelPerformanceSelect(groupColumn string, includeDimensions bool, retryMetricsAvailable bool) (string, []any) { + prefix := "COUNT(DISTINCT channel_id) AS channel_count, " + if includeDimensions { + prefix = fmt.Sprintf("channel_id, model_name, %s AS effective_group, ", groupColumn) + } + + retryExpr := "0 AS retry_log_count, " + args := []any{LogTypeConsume, LogTypeError, LogTypeConsume} + if retryMetricsAvailable { + retryExpr = "SUM(CASE WHEN is_retry = ? OR is_error_retry = ? OR is_empty_retry = ? THEN 1 ELSE 0 END) AS retry_log_count, " + args = append(args, true, true, true) + } + // consumed_quota is an operational projection over the same eligible + // Consume logs as the performance row. It is not a complete billing ledger + // total because known accounting-only adjustments are excluded upstream. + selectClause := prefix + + "COUNT(*) AS observed_count, " + + "SUM(CASE WHEN type = ? THEN 1 ELSE 0 END) AS consume_log_count, " + + "SUM(CASE WHEN type = ? THEN 1 ELSE 0 END) AS error_log_count, " + + "SUM(CASE WHEN type = ? THEN quota ELSE 0 END) AS consumed_quota, " + + retryExpr + if !retryMetricsAvailable { + return selectClause + + "0 AS total_use_time_seconds, " + + "0 AS latency_sample_count, " + + "MAX(created_at) AS last_observed_at", args + } + + args = append(args, + LogTypeConsume, legacyTaskSubmissionPattern, false, false, false, + LogTypeConsume, legacyTaskSubmissionPattern, false, false, false, + ) + return selectClause + + "SUM(CASE WHEN (type <> ? OR other IS NULL OR other NOT LIKE ?) AND is_retry = ? AND is_error_retry = ? AND is_empty_retry = ? THEN use_time ELSE 0 END) AS total_use_time_seconds, " + + "SUM(CASE WHEN (type <> ? OR other IS NULL OR other NOT LIKE ?) AND is_retry = ? AND is_error_retry = ? AND is_empty_retry = ? THEN 1 ELSE 0 END) AS latency_sample_count, " + + "MAX(created_at) AS last_observed_at", args +} + +// GetChannelPerformanceChannels returns only the metadata required to label +// aggregate rows. It is intentionally a separate query because LOG_DB may be +// configured as a different database from the primary application database. +func GetChannelPerformanceChannels(ctx context.Context, ids []int) (map[int]ChannelPerformanceChannel, error) { + result := make(map[int]ChannelPerformanceChannel, len(ids)) + if len(ids) == 0 { + return result, nil + } + if DB == nil { + return nil, fmt.Errorf("primary database is not initialized") + } + + var channels []ChannelPerformanceChannel + err := DB.WithContext(ctx).Model(&Channel{}). + Select("id, name, type, status, response_time, test_time"). + Where("id IN ?", ids).Find(&channels).Error + if err != nil { + return nil, err + } + for _, channel := range channels { + result[channel.ID] = channel + } + return result, nil +} diff --git a/model/channel_performance_test.go b/model/channel_performance_test.go new file mode 100644 index 00000000..162bc95a --- /dev/null +++ b/model/channel_performance_test.go @@ -0,0 +1,207 @@ +package model + +import ( + "context" + "testing" + + "github.com/MAX-API-Next/MAX-API/common" + "github.com/glebarez/sqlite" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +func setupChannelPerformanceTestDB(t *testing.T, retryMetricsReady bool) *gorm.DB { + t.Helper() + + previousDB, previousLogDB := DB, LOG_DB + previousSQLite, previousPostgres, previousMySQL := common.UsingSQLite, common.UsingPostgreSQL, common.UsingMySQL + previousLogSQLType := common.LogSqlType + t.Cleanup(func() { + DB, LOG_DB = previousDB, previousLogDB + common.UsingSQLite, common.UsingPostgreSQL, common.UsingMySQL = previousSQLite, previousPostgres, previousMySQL + common.LogSqlType = previousLogSQLType + initCol() + }) + + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + DB, LOG_DB = db, db + common.UsingSQLite, common.UsingPostgreSQL, common.UsingMySQL = true, false, false + common.LogSqlType = common.DatabaseTypeSQLite + initCol() + require.NoError(t, db.AutoMigrate(&Log{}, &Channel{}, &Option{})) + if retryMetricsReady { + require.NoError(t, markLogRetryMarkerBackfillCompleted()) + } + return db +} + +func TestInitColUsesIndependentLogDatabaseDialect(t *testing.T) { + previousSQLite, previousPostgres, previousMySQL := common.UsingSQLite, common.UsingPostgreSQL, common.UsingMySQL + previousLogSQLType := common.LogSqlType + t.Cleanup(func() { + common.UsingSQLite, common.UsingPostgreSQL, common.UsingMySQL = previousSQLite, previousPostgres, previousMySQL + common.LogSqlType = previousLogSQLType + initCol() + }) + t.Setenv("LOG_SQL_DSN", "separate-log-database") + + common.UsingSQLite, common.UsingPostgreSQL, common.UsingMySQL = false, true, false + common.LogSqlType = common.DatabaseTypeMySQL + initCol() + require.Equal(t, `"group"`, commonGroupCol) + require.Equal(t, "`group`", logGroupCol) + require.Equal(t, "`key`", logKeyCol) + + common.UsingSQLite, common.UsingPostgreSQL, common.UsingMySQL = false, false, true + common.LogSqlType = common.DatabaseTypePostgreSQL + initCol() + require.Equal(t, "`group`", commonGroupCol) + require.Equal(t, `"group"`, logGroupCol) + require.Equal(t, `"key"`, logKeyCol) +} + +func TestQueryLegacyChannelPerformanceExcludesNonAttemptBillingLogs(t *testing.T) { + db := setupChannelPerformanceTestDB(t, true) + + now := int64(1_700_000_000) + require.NoError(t, db.Create(&Channel{Id: 7, Name: "Primary", Type: 1, Status: 1, ResponseTime: 240, TestTime: now}).Error) + require.NoError(t, db.Create(&[]Log{ + {CreatedAt: now - 20, Type: LogTypeConsume, ModelName: "alpha", ChannelId: 7, Group: "prod", Quota: 120, UseTime: 9, Other: `{"retry_log":true}`}, + {CreatedAt: now - 10, Type: LogTypeError, ModelName: "alpha", ChannelId: 7, Group: "prod", Quota: 999, UseTime: 1}, + {CreatedAt: now - 9, Type: LogTypeConsume, ModelName: "alpha", ChannelId: 7, Group: "prod", Quota: 50, UseTime: 5, Content: "Violation fee charged", Other: `{"violation_fee":true}`}, + {CreatedAt: now - 8, Type: LogTypeConsume, ModelName: "alpha", ChannelId: 7, Group: "prod", Quota: 40, Other: `{"task_id":"task-1","pre_consumed_quota":10,"actual_quota":15}`}, + {CreatedAt: now - 7, Type: LogTypeConsume, ModelName: "alpha", ChannelId: 7, Group: "prod", Quota: 80, Other: `{"is_task":true}`}, + {CreatedAt: now - 5, Type: LogTypeConsume, ModelName: "alpha", ChannelId: 7, Group: "prod", Quota: 70, TokenName: "模型测试"}, + {CreatedAt: now - 4, Type: LogTypeConsume, ModelName: "alpha", ChannelId: 7, Group: "prod", Quota: 60, Content: "模型测试"}, + }).Error) + + result, err := QueryLegacyChannelPerformance(context.Background(), ChannelPerformanceQuery{ + StartAt: now - 100, + EndAt: now, + Limit: 10, + }) + require.NoError(t, err) + require.True(t, result.RetryMetricsAvailable) + require.Equal(t, LegacyRetryMetricStateAvailable, result.RetryMetricState) + require.Len(t, result.Rows, 1) + row := result.Rows[0] + require.Equal(t, 7, row.ChannelID) + require.Equal(t, "alpha", row.ModelName) + require.Equal(t, "prod", row.EffectiveGroup) + require.EqualValues(t, 3, row.ObservedCount) + require.EqualValues(t, 2, row.ConsumeLogCount) + require.EqualValues(t, 1, row.ErrorLogCount) + require.EqualValues(t, 200, row.ConsumedQuota) + require.EqualValues(t, 1, row.RetryLogCount) + require.EqualValues(t, 1, row.LatencySampleCount) + require.EqualValues(t, 1, row.TotalUseTimeSeconds) + require.EqualValues(t, 3, result.Summary.ObservedCount) + require.EqualValues(t, 2, result.Summary.ConsumeLogCount) + require.EqualValues(t, 1, result.Summary.ErrorLogCount) + require.EqualValues(t, 200, result.Summary.ConsumedQuota) + require.EqualValues(t, 1, result.Summary.LatencySampleCount) + require.EqualValues(t, 1, result.Summary.TotalUseTimeSeconds) + require.EqualValues(t, 1, result.Summary.ChannelCount) + require.EqualValues(t, now-7, result.Summary.LastObservedAt) + + metadata, err := GetChannelPerformanceChannels(context.Background(), []int{7}) + require.NoError(t, err) + require.Equal(t, "Primary", metadata[7].Name) + require.Equal(t, 240, metadata[7].ResponseTime) +} + +func TestQueryLegacyChannelPerformanceSummaryIsNotLimitedWithRows(t *testing.T) { + db := setupChannelPerformanceTestDB(t, true) + now := int64(1_700_000_000) + logs := make([]Log, 0, 3) + for channelID := 1; channelID <= 3; channelID++ { + logs = append(logs, Log{ + CreatedAt: now - int64(channelID), Type: LogTypeConsume, + ModelName: "alpha", ChannelId: channelID, Group: "prod", Quota: channelID * 10, UseTime: channelID, + }) + } + require.NoError(t, db.Create(&logs).Error) + + result, err := QueryLegacyChannelPerformance(context.Background(), ChannelPerformanceQuery{ + StartAt: now - 100, + EndAt: now, + Limit: 1, + }) + require.NoError(t, err) + require.Len(t, result.Rows, 2, "the row query should retain one extra row for truncation detection") + require.EqualValues(t, 3, result.Summary.ChannelCount) + require.EqualValues(t, 3, result.Summary.ObservedCount) + require.EqualValues(t, 3, result.Summary.ConsumeLogCount) + require.EqualValues(t, 60, result.Summary.ConsumedQuota) + require.EqualValues(t, 3, result.Summary.LatencySampleCount) + require.EqualValues(t, 6, result.Summary.TotalUseTimeSeconds) + require.EqualValues(t, now-1, result.Summary.LastObservedAt) +} + +func TestQueryLegacyChannelPerformancePreservesSignedConsumeQuota(t *testing.T) { + db := setupChannelPerformanceTestDB(t, true) + now := int64(1_700_000_000) + require.NoError(t, db.Create(&[]Log{ + {CreatedAt: now - 3, Type: LogTypeConsume, ModelName: "alpha", ChannelId: 7, Group: "prod", Quota: 100}, + {CreatedAt: now - 2, Type: LogTypeConsume, ModelName: "alpha", ChannelId: 7, Group: "prod", Quota: 0}, + {CreatedAt: now - 1, Type: LogTypeConsume, ModelName: "alpha", ChannelId: 7, Group: "prod", Quota: -25}, + {CreatedAt: now - 1, Type: LogTypeError, ModelName: "alpha", ChannelId: 7, Group: "prod", Quota: 500}, + }).Error) + + result, err := QueryLegacyChannelPerformance(context.Background(), ChannelPerformanceQuery{ + StartAt: now - 100, + EndAt: now, + Limit: 10, + }) + require.NoError(t, err) + require.Len(t, result.Rows, 1) + require.EqualValues(t, 75, result.Rows[0].ConsumedQuota) + require.EqualValues(t, 75, result.Summary.ConsumedQuota) +} + +func TestQueryLegacyChannelPerformanceMarksRetryMetricsUnavailableBeforeBackfill(t *testing.T) { + db := setupChannelPerformanceTestDB(t, false) + now := int64(1_700_000_000) + log := Log{ + CreatedAt: now - 1, Type: LogTypeConsume, ModelName: "alpha", ChannelId: 7, + Group: "prod", UseTime: 5, Other: `{"retry_log":true}`, + } + require.NoError(t, db.Create(&log).Error) + require.NoError(t, db.Model(&Log{}).Where("id = ?", log.Id).Updates(map[string]interface{}{ + "is_retry": false, "is_error_retry": false, "is_empty_retry": false, + }).Error) + + result, err := QueryLegacyChannelPerformance(context.Background(), ChannelPerformanceQuery{ + StartAt: now - 100, + EndAt: now, + Limit: 10, + }) + require.NoError(t, err) + require.False(t, result.RetryMetricsAvailable) + require.Equal(t, LegacyRetryMetricStatePending, result.RetryMetricState) + require.Len(t, result.Rows, 1) + require.Zero(t, result.Rows[0].RetryLogCount) + require.Zero(t, result.Rows[0].LatencySampleCount) + require.Zero(t, result.Rows[0].TotalUseTimeSeconds) + require.Zero(t, result.Summary.RetryLogCount) + require.Zero(t, result.Summary.LatencySampleCount) + require.Zero(t, result.Summary.TotalUseTimeSeconds) +} + +func TestQueryLegacyChannelPerformanceReturnsZeroSummaryWithoutRows(t *testing.T) { + setupChannelPerformanceTestDB(t, true) + + result, err := QueryLegacyChannelPerformance(context.Background(), ChannelPerformanceQuery{ + StartAt: 1_700_000_000, + EndAt: 1_700_000_100, + Limit: 10, + }) + require.NoError(t, err) + require.Empty(t, result.Rows) + require.Zero(t, result.Summary.ChannelCount) + require.Zero(t, result.Summary.ObservedCount) + require.Zero(t, result.Summary.ConsumedQuota) + require.Zero(t, result.Summary.LatencySampleCount) + require.Zero(t, result.Summary.LastObservedAt) +} diff --git a/model/main.go b/model/main.go index e336bd62..84592406 100644 --- a/model/main.go +++ b/model/main.go @@ -84,9 +84,12 @@ func initCol() { case common.DatabaseTypePostgreSQL: logGroupCol = `"group"` logKeyCol = `"key"` + case common.DatabaseTypeMySQL, common.DatabaseTypeSQLite: + logGroupCol = "`group`" + logKeyCol = "`key`" default: - logGroupCol = commonGroupCol - logKeyCol = commonKeyCol + logGroupCol = "`group`" + logKeyCol = "`key`" } } else { // LOG_SQL_DSN 为空时，日志数据库与主数据库相同 @@ -345,6 +348,7 @@ func migrateDB() error { &UserOAuthBinding{}, &AuthFlow{}, &PerfMetric{}, + &PerfMetricFlushReceipt{}, &SystemTask{}, &SystemInstance{}, ) diff --git a/model/model_performance.go b/model/model_performance.go new file mode 100644 index 00000000..9a1ee067 --- /dev/null +++ b/model/model_performance.go @@ -0,0 +1,147 @@ +package model + +import ( + "context" + "fmt" +) + +// ModelPerformanceQuery describes the bounded, read-only legacy log query +// used by Smart Operations Center to aggregate production evidence by model. +type ModelPerformanceQuery struct { + StartAt int64 + EndAt int64 + Hours int + Limit int + ModelName string + Group string +} + +type LegacyModelPerformanceRow struct { + ModelName string `gorm:"column:model_name"` + ChannelCount int64 `gorm:"column:channel_count"` + ObservedCount int64 `gorm:"column:observed_count"` + ConsumeLogCount int64 `gorm:"column:consume_log_count"` + ErrorLogCount int64 `gorm:"column:error_log_count"` + ConsumedQuota int64 `gorm:"column:consumed_quota"` + RetryLogCount int64 `gorm:"column:retry_log_count"` + LatencySampleCount int64 `gorm:"column:latency_sample_count"` + TotalUseTimeSeconds int64 `gorm:"column:total_use_time_seconds"` + LastObservedAt int64 `gorm:"column:last_observed_at"` +} + +type LegacyModelPerformanceSummary struct { + ModelCount int64 `gorm:"column:model_count"` + ChannelCount int64 `gorm:"column:channel_count"` + ObservedCount int64 `gorm:"column:observed_count"` + ConsumeLogCount int64 `gorm:"column:consume_log_count"` + ErrorLogCount int64 `gorm:"column:error_log_count"` + ConsumedQuota int64 `gorm:"column:consumed_quota"` + RetryLogCount int64 `gorm:"column:retry_log_count"` + LatencySampleCount int64 `gorm:"column:latency_sample_count"` + TotalUseTimeSeconds int64 `gorm:"column:total_use_time_seconds"` + LastObservedAt int64 `gorm:"column:last_observed_at"` +} + +type LegacyModelPerformanceResult struct { + Rows []LegacyModelPerformanceRow + Summary LegacyModelPerformanceSummary + RetryMetricsAvailable bool + RetryMetricState string +} + +// QueryLegacyModelPerformance reads existing log rows and aggregates them by +// model. It intentionally shares the channel-performance eligibility rules so +// both views exclude probes, billing adjustments and known non-attempt logs in +// the same way. +func QueryLegacyModelPerformance(ctx context.Context, query ModelPerformanceQuery) (LegacyModelPerformanceResult, error) { + if LOG_DB == nil { + return LegacyModelPerformanceResult{}, fmt.Errorf("log database is not initialized") + } + + limit := query.Limit + if limit <= 0 { + limit = 100 + } + if limit > 200 { + limit = 200 + } + + groupColumn := logGroupCol + if groupColumn == "" { + groupColumn = commonGroupCol + } + + retryMetricState := legacyChannelPerformanceRetryMetricState() + retryMetricsAvailable := retryMetricState == LegacyRetryMetricStateAvailable + rowSelect, rowArgs := legacyModelPerformanceSelect(true, retryMetricsAvailable) + baseQuery := ChannelPerformanceQuery{ + StartAt: query.StartAt, + EndAt: query.EndAt, + ModelName: query.ModelName, + Group: query.Group, + } + + var rows []LegacyModelPerformanceRow + err := legacyChannelPerformanceQuery(ctx, baseQuery, groupColumn). + Select(rowSelect, rowArgs...). + Group("model_name"). + Order("observed_count DESC, last_observed_at DESC, model_name ASC"). + Limit(limit + 1). + Scan(&rows).Error + if err != nil { + return LegacyModelPerformanceResult{}, err + } + + summarySelect, summaryArgs := legacyModelPerformanceSelect(false, retryMetricsAvailable) + var summary LegacyModelPerformanceSummary + if err := legacyChannelPerformanceQuery(ctx, baseQuery, groupColumn). + Select(summarySelect, summaryArgs...). + Scan(&summary).Error; err != nil { + return LegacyModelPerformanceResult{}, err + } + + return LegacyModelPerformanceResult{ + Rows: rows, + Summary: summary, + RetryMetricsAvailable: retryMetricsAvailable, + RetryMetricState: retryMetricState, + }, nil +} + +func legacyModelPerformanceSelect(includeModel bool, retryMetricsAvailable bool) (string, []any) { + prefix := "COUNT(DISTINCT model_name) AS model_count, COUNT(DISTINCT channel_id) AS channel_count, " + if includeModel { + prefix = "model_name, COUNT(DISTINCT channel_id) AS channel_count, " + } + + retryExpr := "0 AS retry_log_count, " + args := []any{LogTypeConsume, LogTypeError, LogTypeConsume} + if retryMetricsAvailable { + retryExpr = "SUM(CASE WHEN is_retry = ? OR is_error_retry = ? OR is_empty_retry = ? THEN 1 ELSE 0 END) AS retry_log_count, " + args = append(args, true, true, true) + } + + // Keep quota on the same eligible-log projection as channel performance; + // this operational total intentionally excludes accounting-only rows. + selectClause := prefix + + "COUNT(*) AS observed_count, " + + "SUM(CASE WHEN type = ? THEN 1 ELSE 0 END) AS consume_log_count, " + + "SUM(CASE WHEN type = ? THEN 1 ELSE 0 END) AS error_log_count, " + + "SUM(CASE WHEN type = ? THEN quota ELSE 0 END) AS consumed_quota, " + + retryExpr + if !retryMetricsAvailable { + return selectClause + + "0 AS total_use_time_seconds, " + + "0 AS latency_sample_count, " + + "MAX(created_at) AS last_observed_at", args + } + + args = append(args, + LogTypeConsume, legacyTaskSubmissionPattern, false, false, false, + LogTypeConsume, legacyTaskSubmissionPattern, false, false, false, + ) + return selectClause + + "SUM(CASE WHEN (type <> ? OR other IS NULL OR other NOT LIKE ?) AND is_retry = ? AND is_error_retry = ? AND is_empty_retry = ? THEN use_time ELSE 0 END) AS total_use_time_seconds, " + + "SUM(CASE WHEN (type <> ? OR other IS NULL OR other NOT LIKE ?) AND is_retry = ? AND is_error_retry = ? AND is_empty_retry = ? THEN 1 ELSE 0 END) AS latency_sample_count, " + + "MAX(created_at) AS last_observed_at", args +} diff --git a/model/model_performance_test.go b/model/model_performance_test.go new file mode 100644 index 00000000..4f358f47 --- /dev/null +++ b/model/model_performance_test.go @@ -0,0 +1,101 @@ +package model + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestQueryLegacyModelPerformanceAggregatesAcrossChannels(t *testing.T) { + db := setupChannelPerformanceTestDB(t, true) + now := int64(1_700_000_000) + + require.NoError(t, db.Create(&[]Log{ + {CreatedAt: now - 20, Type: LogTypeConsume, ModelName: "alpha", ChannelId: 1, Group: "prod", Quota: 120, UseTime: 2}, + {CreatedAt: now - 19, Type: LogTypeError, ModelName: "alpha", ChannelId: 1, Group: "prod", Quota: 999}, + {CreatedAt: now - 18, Type: LogTypeConsume, ModelName: "alpha", ChannelId: 2, Group: "prod", Quota: 80, UseTime: 4, Other: `{"retry_log":true}`}, + {CreatedAt: now - 10, Type: LogTypeConsume, ModelName: "beta", ChannelId: 2, Group: "prod", Quota: 50, UseTime: 3}, + {CreatedAt: now - 9, Type: LogTypeConsume, ModelName: "ignored", ChannelId: 0, Group: "prod", Quota: 1000, UseTime: 8}, + }).Error) + + result, err := QueryLegacyModelPerformance(context.Background(), ModelPerformanceQuery{ + StartAt: now - 100, + EndAt: now, + Limit: 10, + }) + require.NoError(t, err) + require.True(t, result.RetryMetricsAvailable) + require.Len(t, result.Rows, 2) + + alpha := result.Rows[0] + require.Equal(t, "alpha", alpha.ModelName) + require.EqualValues(t, 2, alpha.ChannelCount) + require.EqualValues(t, 3, alpha.ObservedCount) + require.EqualValues(t, 2, alpha.ConsumeLogCount) + require.EqualValues(t, 1, alpha.ErrorLogCount) + require.EqualValues(t, 200, alpha.ConsumedQuota) + require.EqualValues(t, 1, alpha.RetryLogCount) + require.EqualValues(t, 2, alpha.LatencySampleCount) + require.EqualValues(t, 2, alpha.TotalUseTimeSeconds) + require.EqualValues(t, now-18, alpha.LastObservedAt) + + require.Equal(t, "beta", result.Rows[1].ModelName) + require.EqualValues(t, 1, result.Rows[1].ChannelCount) + require.EqualValues(t, 1, result.Rows[1].ObservedCount) + require.EqualValues(t, 50, result.Rows[1].ConsumedQuota) + require.EqualValues(t, 1, result.Rows[1].LatencySampleCount) + + require.EqualValues(t, 2, result.Summary.ModelCount) + require.EqualValues(t, 2, result.Summary.ChannelCount) + require.EqualValues(t, 4, result.Summary.ObservedCount) + require.EqualValues(t, 3, result.Summary.ConsumeLogCount) + require.EqualValues(t, 1, result.Summary.ErrorLogCount) + require.EqualValues(t, 250, result.Summary.ConsumedQuota) + require.EqualValues(t, 1, result.Summary.RetryLogCount) + require.EqualValues(t, 3, result.Summary.LatencySampleCount) + require.EqualValues(t, now-10, result.Summary.LastObservedAt) +} + +func TestQueryLegacyModelPerformanceSummaryIsNotLimitedWithRows(t *testing.T) { + db := setupChannelPerformanceTestDB(t, true) + now := int64(1_700_000_000) + logs := []Log{ + {CreatedAt: now - 1, Type: LogTypeConsume, ModelName: "alpha", ChannelId: 1, Group: "prod", Quota: 10}, + {CreatedAt: now - 2, Type: LogTypeConsume, ModelName: "beta", ChannelId: 2, Group: "prod", Quota: 20}, + } + require.NoError(t, db.Create(&logs).Error) + + result, err := QueryLegacyModelPerformance(context.Background(), ModelPerformanceQuery{ + StartAt: now - 100, + EndAt: now, + Limit: 1, + }) + require.NoError(t, err) + require.Len(t, result.Rows, 2, "the row query should retain one extra row for truncation detection") + require.EqualValues(t, 2, result.Summary.ModelCount) + require.EqualValues(t, 2, result.Summary.ObservedCount) + require.EqualValues(t, 30, result.Summary.ConsumedQuota) +} + +func TestQueryLegacyModelPerformanceSupportsModelAndGroupFilters(t *testing.T) { + db := setupChannelPerformanceTestDB(t, true) + now := int64(1_700_000_000) + require.NoError(t, db.Create(&[]Log{ + {CreatedAt: now - 2, Type: LogTypeConsume, ModelName: "alpha", ChannelId: 1, Group: "prod"}, + {CreatedAt: now - 1, Type: LogTypeConsume, ModelName: "alpha", ChannelId: 2, Group: "staging"}, + }).Error) + + result, err := QueryLegacyModelPerformance(context.Background(), ModelPerformanceQuery{ + StartAt: now - 100, + EndAt: now, + Limit: 10, + ModelName: "alpha", + Group: "prod", + }) + require.NoError(t, err) + require.Len(t, result.Rows, 1) + require.EqualValues(t, 1, result.Rows[0].ChannelCount) + require.EqualValues(t, 1, result.Summary.ModelCount) + require.EqualValues(t, 1, result.Summary.ChannelCount) +} diff --git a/model/perf_metric.go b/model/perf_metric.go index 1667adfb..9480a6cd 100644 --- a/model/perf_metric.go +++ b/model/perf_metric.go @@ -3,6 +3,7 @@ package model import ( "time" + "github.com/google/uuid" "gorm.io/gorm" "gorm.io/gorm/clause" ) @@ -22,15 +23,34 @@ type PerfMetric struct { GenerationMs int64 `json:"-" gorm:"default:0"` } +// PerfMetricFlushReceipt makes a node-local bucket commit idempotent across +// process restarts and ambiguous database responses. It is intentionally kept +// separate from PerfMetric so existing metric rows and query contracts remain +// unchanged. +type PerfMetricFlushReceipt struct { + Id int64 `gorm:"primaryKey"` + ReceiptKey string `gorm:"size:64;uniqueIndex;not null"` + ClaimToken string `gorm:"size:36;not null"` + CreatedAt int64 `gorm:"not null;index"` +} + +func (PerfMetricFlushReceipt) TableName() string { + return "perf_metric_flush_receipts" +} + func (PerfMetric) TableName() string { return "perf_metrics" } func UpsertPerfMetric(metric *PerfMetric) error { + return upsertPerfMetric(DB, metric) +} + +func upsertPerfMetric(db *gorm.DB, metric *PerfMetric) error { if metric == nil || metric.RequestCount == 0 { return nil } - return DB.Clauses(clause.OnConflict{ + return db.Clauses(clause.OnConflict{ Columns: []clause.Column{ {Name: "model_name"}, {Name: "group"}, @@ -48,12 +68,56 @@ func UpsertPerfMetric(metric *PerfMetric) error { }).Create(metric).Error } +// CommitPerfMetricBucket atomically records a durable receipt and applies the +// bucket counters once. A repeated receipt is a successful no-op, allowing a +// recovered Redis bucket to be cleaned without incrementing PerfMetric twice. +func CommitPerfMetricBucket(receiptKey string, metric *PerfMetric) (bool, error) { + if metric == nil || metric.RequestCount == 0 { + return false, nil + } + if receiptKey == "" { + return false, gorm.ErrInvalidData + } + + applied := false + err := DB.Transaction(func(tx *gorm.DB) error { + claimToken := uuid.NewString() + result := tx.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "receipt_key"}}, + DoNothing: true, + }).Create(&PerfMetricFlushReceipt{ + ReceiptKey: receiptKey, + ClaimToken: claimToken, + CreatedAt: time.Now().Unix(), + }) + if result.Error != nil { + return result.Error + } + var storedReceipt PerfMetricFlushReceipt + if err := tx.Select("claim_token").Where("receipt_key = ?", receiptKey).Take(&storedReceipt).Error; err != nil { + return err + } + // MySQL can report a row as affected for its no-op duplicate-key + // update when clientFoundRows is enabled. Ownership is therefore proven + // by a per-attempt claim rather than driver-specific RowsAffected values. + if storedReceipt.ClaimToken != claimToken { + return nil + } + if err := upsertPerfMetric(tx, metric); err != nil { + return err + } + applied = true + return nil + }) + return applied, err +} + func GetPerfMetrics(modelName string, group string, startTs int64, endTs int64) ([]PerfMetric, error) { var metrics []PerfMetric query := DB.Model(&PerfMetric{}). Where("model_name = ? AND bucket_ts >= ? AND bucket_ts <= ?", modelName, startTs, endTs) if group != "" { - query = query.Where(commonGroupCol+" = ?", group) + query = query.Where(map[string]interface{}{"group": group}) } err := query.Order("bucket_ts ASC").Find(&metrics).Error return metrics, err @@ -66,18 +130,20 @@ type PerfMetricSummary struct { TotalLatencyMs int64 `json:"total_latency_ms"` OutputTokens int64 `json:"output_tokens"` GenerationMs int64 `json:"generation_ms"` + MinBucketTs int64 `json:"min_bucket_ts"` + MaxBucketTs int64 `json:"max_bucket_ts"` } func GetPerfMetricsSummaryAll(startTs int64, endTs int64, groups []string) ([]PerfMetricSummary, error) { var summaries []PerfMetricSummary query := DB.Model(&PerfMetric{}). - Select("model_name, SUM(request_count) as request_count, SUM(success_count) as success_count, SUM(total_latency_ms) as total_latency_ms, SUM(output_tokens) as output_tokens, SUM(generation_ms) as generation_ms"). + Select("model_name, SUM(request_count) as request_count, SUM(success_count) as success_count, SUM(total_latency_ms) as total_latency_ms, SUM(output_tokens) as output_tokens, SUM(generation_ms) as generation_ms, MIN(bucket_ts) as min_bucket_ts, MAX(bucket_ts) as max_bucket_ts"). Where("bucket_ts >= ? AND bucket_ts <= ?", startTs, endTs) if groups != nil { if len(groups) == 0 { return summaries, nil } - query = query.Where(commonGroupCol+" IN ?", groups) + query = query.Where(map[string]interface{}{"group": groups}) } err := query. Group("model_name"). @@ -93,6 +159,28 @@ func DeletePerfMetricsBefore(cutoffTs int64) error { return DB.Where("bucket_ts < ?", cutoffTs).Delete(&PerfMetric{}).Error } +func DeletePerfMetricFlushReceiptsBefore(cutoffTs int64) error { + if cutoffTs <= 0 { + return nil + } + return DB.Where("created_at < ?", cutoffTs).Delete(&PerfMetricFlushReceipt{}).Error +} + +func GetPerfMetricFlushReceiptKeys(receiptKeys []string) (map[string]struct{}, error) { + result := make(map[string]struct{}, len(receiptKeys)) + if len(receiptKeys) == 0 { + return result, nil + } + var receipts []PerfMetricFlushReceipt + if err := DB.Select("receipt_key").Where("receipt_key IN ?", receiptKeys).Find(&receipts).Error; err != nil { + return nil, err + } + for _, receipt := range receipts { + result[receipt.ReceiptKey] = struct{}{} + } + return result, nil +} + func PerfMetricStartTime(hours int) int64 { if hours <= 0 { hours = 24 diff --git a/pkg/perf_metrics/flush.go b/pkg/perf_metrics/flush.go index ae34fedd..9de14abf 100644 --- a/pkg/perf_metrics/flush.go +++ b/pkg/perf_metrics/flush.go @@ -1,6 +1,7 @@ package perfmetrics import ( + "context" "fmt" "strconv" "time" @@ -14,60 +15,146 @@ func flushLoop() { for { interval := perf_metrics_setting.GetFlushIntervalMinutes() time.Sleep(time.Duration(interval) * time.Minute) - setting := perf_metrics_setting.GetSetting() - if !setting.Enabled { - continue - } - flushCompletedBuckets() - cleanupExpiredMetrics(setting.RetentionDays) + flushOnce() } } +func flushOnce() { + setting := perf_metrics_setting.GetSetting() + // Disabling collection stops new samples at Record, but buckets that + // were already accepted still need to be drained and committed. Skipping + // the flush here would strand those samples in memory and leave committed + // Redis snapshots around until their TTL expires. + flushCompletedBuckets() + cleanupCommittedRedisBuckets() + cleanupExpiredMetrics(setting.RetentionDays) +} + func flushCompletedBuckets() { - currentBucket := bucketStart(time.Now().Unix()) + now := time.Now().Unix() hotBuckets.Range(func(key, value any) bool { k := key.(bucketKey) - if k.bucketTs >= currentBucket { + bucketSeconds := k.bucketSeconds + if bucketSeconds <= 0 { + bucketSeconds = perf_metrics_setting.GetBucketSeconds() + } + if bucketSeconds <= 0 { + bucketSeconds = 3600 + } + if k.bucketTs+bucketSeconds > now { return true } bucket := value.(*atomicBucket) - drained := bucket.drain() + drained := bucket.closeAndDrain() + hotBuckets.CompareAndDelete(k, bucket) if drained.requestCount == 0 { - deleteOldEmptyBucket(k, key) + cleanupFlushedBucket(k) return true } - err := model.UpsertPerfMetric(&model.PerfMetric{ - ModelName: k.model, - Group: k.group, - BucketTs: k.bucketTs, - RequestCount: drained.requestCount, - SuccessCount: drained.successCount, - TotalLatencyMs: drained.totalLatencyMs, - TtftSumMs: drained.ttftSumMs, - TtftCount: drained.ttftCount, - OutputTokens: drained.outputTokens, - GenerationMs: drained.generationMs, - }) + _, err := persistBucket(k, drained) if err != nil { - bucket.addCounters(drained) + requeueCounters(k, drained) common.SysError(fmt.Sprintf("failed to flush perf metric bucket model=%s group=%s bucket=%d: %s", k.model, k.group, k.bucketTs, err.Error())) return true } - deleteOldEmptyBucket(k, key) + committedBuckets.Store(k, struct{}{}) + cleanupFlushedBucket(k) return true }) } -func deleteOldEmptyBucket(k bucketKey, rawKey any) { - if k.bucketTs < bucketStart(time.Now().Add(-24*time.Hour).Unix()) { - hotBuckets.Delete(rawKey) +func persistBucket(key bucketKey, value counters) (bool, error) { + return model.CommitPerfMetricBucket(perfMetricReceiptKey(key), &model.PerfMetric{ + ModelName: key.model, + Group: key.group, + BucketTs: key.bucketTs, + RequestCount: value.requestCount, + SuccessCount: value.successCount, + TotalLatencyMs: value.totalLatencyMs, + TtftSumMs: value.ttftSumMs, + TtftCount: value.ttftCount, + OutputTokens: value.outputTokens, + GenerationMs: value.generationMs, + }) +} + +func perfMetricReceiptKey(key bucketKey) string { + identity := fmt.Sprintf("%s\x00%s\x00%s\x00%d\x00%d", key.node, key.model, key.group, key.bucketTs, key.bucketSeconds) + return fmt.Sprintf("%x", common.Sha256Raw([]byte(identity))) +} + +func requeueCounters(key bucketKey, value counters) { + for { + actual, _ := hotBuckets.LoadOrStore(key, &atomicBucket{}) + if actual.(*atomicBucket).addCounters(value) { + return + } + hotBuckets.CompareAndDelete(key, actual) + } +} + +func cleanupFlushedBucket(key bucketKey) { + if err := deleteRedisBucket(key); err != nil { + common.SysError(fmt.Sprintf("failed to cleanup flushed perf metric bucket model=%s group=%s bucket=%d: %s", key.model, key.group, key.bucketTs, err.Error())) + return } + committedBuckets.Delete(key) + clearProjectionGap(key) + removeClosedBucket(key) +} + +func cleanupCommittedRedisBuckets() { + committedBuckets.Range(func(key, _ any) bool { + if err := deleteRedisBucket(key.(bucketKey)); err == nil { + committedBuckets.Delete(key) + clearProjectionGap(key.(bucketKey)) + removeClosedBucket(key.(bucketKey)) + } + return true + }) +} + +func removeClosedBucket(key bucketKey) { + value, ok := hotBuckets.Load(key) + if !ok { + return + } + bucket := value.(*atomicBucket) + bucket.mu.Lock() + closed := bucket.closed + bucket.mu.Unlock() + if closed { + hotBuckets.CompareAndDelete(key, bucket) + } +} + +func deleteRedisBucket(key bucketKey) error { + if !common.RedisEnabled || common.RDB == nil { + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + redisKey := redisBucketKey(key) + if err := common.RDB.HSet(ctx, redisKey, "flushed", 1).Err(); err != nil { + return err + } + pipe := common.RDB.TxPipeline() + pipe.ZRem(ctx, redisActiveBucketIndex, redisKey) + pipe.ZRem(ctx, redisNodeBucketIndex(key.node), redisKey) + pipe.ZRem(ctx, redisModelBucketIndex(key.model), redisKey) + pipe.Del(ctx, redisKey) + _, err := pipe.Exec(ctx) + return err } func cleanupExpiredMetrics(retentionDays int) { + receiptCutoff := time.Now().Add(-redisBucketTTL - time.Hour).Unix() + if err := model.DeletePerfMetricFlushReceiptsBefore(receiptCutoff); err != nil { + common.SysError("failed to cleanup performance metric flush receipts: " + err.Error()) + } if retentionDays <= 0 { return } diff --git a/pkg/perf_metrics/metrics.go b/pkg/perf_metrics/metrics.go index 620fddf1..789fe86f 100644 --- a/pkg/perf_metrics/metrics.go +++ b/pkg/perf_metrics/metrics.go @@ -2,26 +2,89 @@ package perfmetrics import ( "context" + "crypto/sha256" "fmt" "math" "sort" + "strconv" "sync" + "sync/atomic" "time" "github.com/MAX-API-Next/MAX-API/common" "github.com/MAX-API-Next/MAX-API/model" relaycommon "github.com/MAX-API-Next/MAX-API/relay/common" "github.com/MAX-API-Next/MAX-API/setting/perf_metrics_setting" + "github.com/go-redis/redis/v8" ) var hotBuckets sync.Map +// committedBuckets suppresses a just-committed Redis snapshot in this +// process until cleanup succeeds. The durable flush receipt is the +// cross-restart guard; this local marker keeps the normal DB-success/ +// Redis-cleanup window from being queried twice before cleanup finishes. +var committedBuckets sync.Map + +// projectionGapBuckets tracks active local buckets whose latest absolute +// snapshot could not be published to Redis. The node health projection makes +// this quality loss visible to queries served by other instances after Redis +// recovers. +var projectionGapBuckets sync.Map +var projectionGapCount atomic.Int64 +var projectionDegradedUntil atomic.Int64 + +const ( + redisActiveBucketIndex = "perf:v2:active" + redisProjectionHealthIndex = "perf:v2:projection-health" + redisBucketTTL = 48 * time.Hour + projectionHealthInterval = 5 * time.Second + projectionHealthyTTL = 30 * time.Second +) + +var redisBucketSnapshotScript = redis.NewScript(` +redis.call('HSET', KEYS[1], + 'model', ARGV[1], + 'group', ARGV[2], + 'node', ARGV[3], + 'bucket_ts', ARGV[4], + 'bucket_seconds', ARGV[5]) +local names = {'req', 'ok', 'lat', 'ttft', 'ttft_n', 'out', 'gen_ms'} +for i = 1, #names do + local incoming = tonumber(ARGV[i + 5]) or 0 + local current = tonumber(redis.call('HGET', KEYS[1], names[i]) or '0') + if incoming > current then + redis.call('HSET', KEYS[1], names[i], incoming) + end +end +redis.call('EXPIRE', KEYS[1], tonumber(ARGV[13])) +redis.call('ZADD', KEYS[2], tonumber(ARGV[14]), KEYS[1]) +redis.call('ZADD', KEYS[3], tonumber(ARGV[14]), KEYS[1]) +redis.call('ZADD', KEYS[4], tonumber(ARGV[14]), KEYS[1]) +redis.call('EXPIRE', KEYS[2], tonumber(ARGV[15])) +redis.call('EXPIRE', KEYS[3], tonumber(ARGV[15])) +redis.call('EXPIRE', KEYS[4], tonumber(ARGV[15])) +return 1 +`) + // seriesSchema is a stable client cache/schema marker. Do not change it when // hiding fields or making response-only privacy hardening changes. const seriesSchema = "dbcd0a3c01b55203" func Init() { + if common.RedisEnabled && common.RDB != nil && currentNodeName() == "unnamed" { + common.SysError("performance metrics Redis recovery requires a unique stable NODE_NAME per running instance") + } + if recovered, err := recoverRedisBucketsForNode(); err != nil { + common.SysError("failed to recover active performance metric buckets from Redis: " + err.Error()) + } else if recovered > 0 { + common.SysLog(fmt.Sprintf("recovered %d active performance metric buckets from Redis", recovered)) + } + if err := restoreProjectionHealth(); err != nil { + common.SysError("failed to restore performance projection health: " + err.Error()) + } go flushLoop() + go projectionHealthLoop() } func RecordRelaySample(info *relaycommon.RelayInfo, success bool, outputTokens int64) { @@ -66,17 +129,50 @@ func Record(sample Sample) { sample.LatencyMs = 0 } + bucketSeconds := perf_metrics_setting.GetBucketSeconds() + if bucketSeconds <= 0 { + bucketSeconds = 3600 + } + now := time.Now().Unix() key := bucketKey{ - model: sample.Model, - group: sample.Group, - bucketTs: bucketStart(time.Now().Unix()), + model: sample.Model, + group: sample.Group, + node: currentNodeName(), + bucketTs: bucketStartFor(now, bucketSeconds), + bucketSeconds: bucketSeconds, + } + projectToRedis := common.RedisEnabled && common.RDB != nil + for { + actual, _ := hotBuckets.LoadOrStore(key, &atomicBucket{}) + bucket := actual.(*atomicBucket) + if !projectToRedis { + if bucket.add(sample) { + break + } + hotBuckets.CompareAndDelete(key, actual) + continue + } + snapshot, version, added := bucket.addForProjection(sample) + if added { + success := recordRedis(key, snapshot) + setProjectionGap(key, bucket.finishProjection(version, success)) + break + } + hotBuckets.CompareAndDelete(key, actual) } - actual, _ := hotBuckets.LoadOrStore(key, &atomicBucket{}) - actual.(*atomicBucket).add(sample) - recordRedis(key, sample) } func Query(params QueryParams) (QueryResult, error) { + result, err := QueryDetailed(params) + if err != nil { + return QueryResult{}, err + } + return result.QueryResult, nil +} + +func QueryDetailed(params QueryParams) (DetailedQueryResult, error) { + setting := perf_metrics_setting.GetSetting() + bucketSeconds := perf_metrics_setting.GetBucketSeconds() if params.Hours <= 0 { params.Hours = 24 } @@ -87,15 +183,20 @@ func Query(params QueryParams) (QueryResult, error) { startTs := endTs - int64(params.Hours)*3600 merged := map[bucketKey]counters{} - rows, err := model.GetPerfMetrics(params.Model, params.Group, startTs, endTs) + metricStartTs := bucketStartFor(startTs, bucketSeconds) + rows, err := model.GetPerfMetrics(params.Model, params.Group, metricStartTs, endTs) if err != nil { - return QueryResult{}, err + return DetailedQueryResult{}, err } for _, row := range rows { + if !matchesAllowedGroup(row.Group, params.AllowedGroups) { + continue + } mergeCounters(merged, bucketKey{ - model: row.ModelName, - group: row.Group, - bucketTs: row.BucketTs, + model: row.ModelName, + group: row.Group, + bucketTs: row.BucketTs, + bucketSeconds: 0, }, counters{ requestCount: row.RequestCount, successCount: row.SuccessCount, @@ -107,19 +208,20 @@ func Query(params QueryParams) (QueryResult, error) { }) } - hotBuckets.Range(func(key, value any) bool { - k := key.(bucketKey) - if k.model != params.Model || k.bucketTs < startTs || k.bucketTs > endTs { - return true - } - if params.Group != "" && k.group != params.Group { - return true - } - mergeCounters(merged, k, value.(*atomicBucket).snapshot()) - return true + activeState := mergeActiveBuckets(merged, activeBucketFilter{ + model: params.Model, + group: params.Group, + allowedGroups: allowedGroupSet(params.AllowedGroups), + startTs: metricStartTs, + endTs: endTs, }) - return buildQueryResult(params.Model, merged), nil + queryResult := buildQueryResult(params.Model, merged) + return DetailedQueryResult{ + QueryResult: queryResult, + Coverage: buildCoverage(startTs, endTs, merged, bucketSeconds), + CollectionState: resolveCollectionState(setting.Enabled, len(queryResult.Groups) > 0, activeState), + }, nil } func QuerySummaryAll(hours int, groups []string) (SummaryAllResult, error) { @@ -131,14 +233,45 @@ func QuerySummaryAll(hours int, groups []string) (SummaryAllResult, error) { } endTs := time.Now().Unix() startTs := endTs - int64(hours)*3600 - allowedGroups := allowedGroupSet(groups) + return QuerySummaryAllRange(startTs, endTs, groups) +} - rows, err := model.GetPerfMetricsSummaryAll(startTs, endTs, groups) +// QuerySummaryAllRange aggregates bucketed model metrics whose bucket start is +// inside the caller-owned time range. Call QuerySummaryAllRangeDetailed when +// an operational client also needs the requested range, selected bucket +// coverage, bucket size, and collection state. +func QuerySummaryAllRange(startTs int64, endTs int64, groups []string) (SummaryAllResult, error) { + result, err := QuerySummaryAllRangeDetailed(startTs, endTs, groups) if err != nil { return SummaryAllResult{}, err } + return SummaryAllResult{Models: result.Models}, nil +} + +func QuerySummaryAllRangeDetailed(startTs int64, endTs int64, groups []string) (DetailedSummaryAllResult, error) { + setting := perf_metrics_setting.GetSetting() + bucketSeconds := perf_metrics_setting.GetBucketSeconds() + if endTs <= startTs { + return DetailedSummaryAllResult{ + Coverage: QueryCoverage{ + RequestedStartAt: startTs, + RequestedEndAt: endTs, + BucketSeconds: bucketSeconds, + GranularityState: CoverageGranularityUnknown, + }, + CollectionState: resolveCollectionState(setting.Enabled, false, activeCollectionHealthy), + }, nil + } + allowedGroups := allowedGroupSet(groups) + + metricStartTs := bucketStartFor(startTs, bucketSeconds) + rows, err := model.GetPerfMetricsSummaryAll(metricStartTs, endTs, groups) + if err != nil { + return DetailedSummaryAllResult{}, err + } totals := map[string]counters{} + coverageBuckets := map[bucketKey]struct{}{} for _, row := range rows { totals[row.ModelName] = counters{ requestCount: row.RequestCount, @@ -147,21 +280,19 @@ func QuerySummaryAll(hours int, groups []string) (SummaryAllResult, error) { outputTokens: row.OutputTokens, generationMs: row.GenerationMs, } + coverageBuckets[bucketKey{model: row.ModelName, bucketTs: row.MinBucketTs}] = struct{}{} + coverageBuckets[bucketKey{model: row.ModelName, bucketTs: row.MaxBucketTs}] = struct{}{} } - hotBuckets.Range(func(key, value any) bool { - k := key.(bucketKey) - if k.bucketTs < startTs || k.bucketTs > endTs { - return true - } - if allowedGroups != nil { - if _, ok := allowedGroups[k.group]; !ok { - return true - } - } - snap := value.(*atomicBucket).snapshot() + active := map[bucketKey]counters{} + activeState := mergeActiveBuckets(active, activeBucketFilter{ + allowedGroups: allowedGroups, + startTs: metricStartTs, + endTs: endTs, + }) + for k, snap := range active { if snap.requestCount == 0 { - return true + continue } cur := totals[k.model] cur.requestCount += snap.requestCount @@ -170,8 +301,8 @@ func QuerySummaryAll(hours int, groups []string) (SummaryAllResult, error) { cur.outputTokens += snap.outputTokens cur.generationMs += snap.generationMs totals[k.model] = cur - return true - }) + coverageBuckets[k] = struct{}{} + } models := make([]ModelSummary, 0, len(totals)) for name, total := range totals { @@ -196,7 +327,12 @@ func QuerySummaryAll(hours int, groups []string) (SummaryAllResult, error) { return models[i].RequestCount > models[j].RequestCount }) - return SummaryAllResult{Models: models}, nil + coverage := buildCoverageFromBucketSet(startTs, endTs, coverageBuckets, bucketSeconds) + return DetailedSummaryAllResult{ + Models: models, + Coverage: coverage, + CollectionState: resolveCollectionState(setting.Enabled, len(models) > 0, activeState), + }, nil } func allowedGroupSet(groups []string) map[string]struct{} { @@ -210,14 +346,29 @@ func allowedGroupSet(groups []string) map[string]struct{} { return allowed } -func bucketStart(ts int64) int64 { - bucketSeconds := perf_metrics_setting.GetBucketSeconds() +func bucketStartFor(ts int64, bucketSeconds int64) int64 { if bucketSeconds <= 0 { bucketSeconds = 3600 } return ts - (ts % bucketSeconds) } +func bucketStart(ts int64) int64 { + return bucketStartFor(ts, perf_metrics_setting.GetBucketSeconds()) +} + +func matchesAllowedGroup(group string, groups []string) bool { + if groups == nil { + return true + } + for _, allowed := range groups { + if group == allowed { + return true + } + } + return false +} + func mergeCounters(merged map[bucketKey]counters, key bucketKey, value counters) { if value.requestCount == 0 { return @@ -242,7 +393,15 @@ func buildQueryResult(modelName string, merged map[bucketKey]counters) QueryResu if _, ok := groupBuckets[key.group]; !ok { groupBuckets[key.group] = map[int64]counters{} } - groupBuckets[key.group][key.bucketTs] = value + bucket := groupBuckets[key.group][key.bucketTs] + bucket.requestCount += value.requestCount + bucket.successCount += value.successCount + bucket.totalLatencyMs += value.totalLatencyMs + bucket.ttftSumMs += value.ttftSumMs + bucket.ttftCount += value.ttftCount + bucket.outputTokens += value.outputTokens + bucket.generationMs += value.generationMs + groupBuckets[key.group][key.bucketTs] = bucket } groups := make([]string, 0, len(groupBuckets)) @@ -289,10 +448,139 @@ func buildQueryResult(modelName string, merged map[bucketKey]counters) QueryResu return QueryResult{ ModelName: modelName, SeriesSchema: seriesSchema, + Summary: buildAggregateResult(merged), Groups: results, } } +func buildAggregateResult(merged map[bucketKey]counters) AggregateResult { + total := counters{} + buckets := map[int64]counters{} + for key, value := range merged { + if value.requestCount == 0 { + continue + } + total.requestCount += value.requestCount + total.successCount += value.successCount + total.totalLatencyMs += value.totalLatencyMs + total.ttftSumMs += value.ttftSumMs + total.ttftCount += value.ttftCount + total.outputTokens += value.outputTokens + total.generationMs += value.generationMs + bucket := buckets[key.bucketTs] + bucket.requestCount += value.requestCount + bucket.successCount += value.successCount + bucket.totalLatencyMs += value.totalLatencyMs + bucket.ttftSumMs += value.ttftSumMs + bucket.ttftCount += value.ttftCount + bucket.outputTokens += value.outputTokens + bucket.generationMs += value.generationMs + buckets[key.bucketTs] = bucket + } + + timestamps := make([]int64, 0, len(buckets)) + for ts := range buckets { + timestamps = append(timestamps, ts) + } + sort.Slice(timestamps, func(i, j int) bool { return timestamps[i] < timestamps[j] }) + series := make([]BucketPoint, 0, len(timestamps)) + for _, ts := range timestamps { + series = append(series, bucketPoint(ts, buckets[ts])) + } + + return AggregateResult{ + AvgTtftMs: avg(total.ttftSumMs, total.ttftCount), + AvgLatencyMs: avg(total.totalLatencyMs, total.requestCount), + SuccessRate: successRate(total), + AvgTps: avgTps(total), + Series: series, + } +} + +func buildCoverage(startTs int64, endTs int64, merged map[bucketKey]counters, bucketSeconds int64) QueryCoverage { + buckets := make(map[bucketKey]struct{}, len(merged)) + for key, value := range merged { + if value.requestCount > 0 { + buckets[key] = struct{}{} + } + } + return buildCoverageFromBucketSet(startTs, endTs, buckets, bucketSeconds) +} + +func buildCoverageFromBucketSet(startTs int64, endTs int64, buckets map[bucketKey]struct{}, bucketSeconds int64) QueryCoverage { + if bucketSeconds <= 0 { + bucketSeconds = 3600 + } + coverage := QueryCoverage{ + RequestedStartAt: startTs, + RequestedEndAt: endTs, + BucketSeconds: bucketSeconds, + GranularityState: CoverageGranularityUnknown, + } + if len(buckets) == 0 { + return coverage + } + minTs, maxEndTs := int64(0), int64(0) + hasBucket := false + hasUnknown := false + knownGranularities := map[int64]struct{}{} + for key := range buckets { + ts := key.bucketTs + if !hasBucket || ts < minTs { + minTs = ts + } + span := key.bucketSeconds + if span <= 0 { + hasUnknown = true + span = bucketSeconds + } else { + knownGranularities[span] = struct{}{} + } + if end := ts + span; !hasBucket || end > maxEndTs { + maxEndTs = end + } + hasBucket = true + } + switch { + case hasUnknown && len(knownGranularities) == 0: + coverage.GranularityState = CoverageGranularityUnknown + case hasUnknown || len(knownGranularities) > 1: + coverage.GranularityState = CoverageGranularityMixed + default: + coverage.GranularityState = CoverageGranularityKnown + for known := range knownGranularities { + coverage.BucketSeconds = known + } + } + coverage.BucketStartAt = minTs + coverage.BucketEndAt = min(maxEndTs, endTs) + coverage.Approximate = minTs != startTs || coverage.BucketEndAt != endTs || coverage.GranularityState != CoverageGranularityKnown + return coverage +} + +type activeCollectionHealth int + +const ( + activeCollectionHealthy activeCollectionHealth = iota + activeCollectionFailed +) + +func resolveCollectionState(enabled bool, hasSamples bool, activeState activeCollectionHealth) CollectionState { + if !enabled { + return CollectionStateDisabled + } + if activeState == activeCollectionFailed && !hasSamples { + return CollectionStateQueryFailed + } + if activeState != activeCollectionHealthy && hasSamples { + return CollectionStatePartial + } + if !hasSamples { + return CollectionStateNoSamples + } + return CollectionStateAvailable +} + func bucketPoint(ts int64, value counters) BucketPoint { return BucketPoint{ Ts: ts, @@ -324,52 +612,412 @@ func avgTps(value counters) float64 { return float64(value.outputTokens) / (float64(value.generationMs) / 1000) } -func recordRedis(key bucketKey, sample Sample) { +type activeBucketFilter struct { + model string + group string + allowedGroups map[string]struct{} + startTs int64 + endTs int64 +} + +func mergeActiveBuckets(merged map[bucketKey]counters, filter activeBucketFilter) activeCollectionHealth { if !common.RedisEnabled || common.RDB == nil { + mergeMemoryBuckets(merged, &hotBuckets, filter, false) + return activeCollectionHealthy + } + + redisBuckets, err := loadRedisActiveBuckets(filter) + if err != nil { + mergeMemoryBuckets(merged, &hotBuckets, filter, false) + return activeCollectionFailed + } + for key, value := range redisBuckets { + mergeCounters(merged, key, value) + } + // The local bucket mirrors the same node-specific Redis bucket. Merge its + // monotonic counters by maximum rather than summing, so an ambiguous Redis + // write acknowledgement cannot double count the same sample. + mergeMemoryBuckets(merged, &hotBuckets, filter, true) + if localProjectionDegraded() { + return activeCollectionFailed + } + degraded, err := loadProjectionHealth() + if err != nil || degraded { + return activeCollectionFailed + } + return activeCollectionHealthy +} + +func mergeMemoryBuckets(merged map[bucketKey]counters, buckets *sync.Map, filter activeBucketFilter, mergeReplica bool) { + buckets.Range(func(key, value any) bool { + k := key.(bucketKey) + if !matchesActiveBucket(k, filter) { + return true + } + snapshot := value.(*atomicBucket).snapshot() + if mergeReplica { + mergeCounterReplica(merged, k, snapshot) + } else { + mergeCounters(merged, k, snapshot) + } + return true + }) +} + +func mergeCounterReplica(merged map[bucketKey]counters, key bucketKey, value counters) { + if value.requestCount == 0 { return } + current := merged[key] + current.requestCount = max(current.requestCount, value.requestCount) + current.successCount = max(current.successCount, value.successCount) + current.totalLatencyMs = max(current.totalLatencyMs, value.totalLatencyMs) + current.ttftSumMs = max(current.ttftSumMs, value.ttftSumMs) + current.ttftCount = max(current.ttftCount, value.ttftCount) + current.outputTokens = max(current.outputTokens, value.outputTokens) + current.generationMs = max(current.generationMs, value.generationMs) + merged[key] = current +} + +func matchesActiveBucket(key bucketKey, filter activeBucketFilter) bool { + if key.bucketTs < filter.startTs || key.bucketTs > filter.endTs { + return false + } + if filter.model != "" && key.model != filter.model { + return false + } + if filter.group != "" && key.group != filter.group { + return false + } + if filter.allowedGroups != nil { + _, ok := filter.allowedGroups[key.group] + return ok + } + return true +} + +func recordRedis(key bucketKey, value counters) bool { + if !common.RedisEnabled || common.RDB == nil { + return false + } ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() redisKey := redisBucketKey(key) - pipe := common.RDB.TxPipeline() - pipe.HIncrBy(ctx, redisKey, "req", 1) - if sample.Success { - pipe.HIncrBy(ctx, redisKey, "ok", 1) + expiresAt := time.Now().Add(redisBucketTTL).Unix() + if err := redisBucketSnapshotScript.Run(ctx, common.RDB, []string{ + redisKey, + redisActiveBucketIndex, + redisNodeBucketIndex(key.node), + redisModelBucketIndex(key.model), + }, + key.model, + key.group, + key.node, + key.bucketTs, + key.bucketSeconds, + value.requestCount, + value.successCount, + value.totalLatencyMs, + value.ttftSumMs, + value.ttftCount, + value.outputTokens, + value.generationMs, + int64(redisBucketTTL/time.Second), + expiresAt, + int64((redisBucketTTL+time.Hour)/time.Second), + ).Err(); err != nil { + return false + } + return true +} + +func loadRedisActiveBuckets(filter activeBucketFilter) (map[bucketKey]counters, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + index := redisActiveBucketIndex + if filter.model != "" { + index = redisModelBucketIndex(filter.model) + } + keys, err := activeRedisKeys(ctx, index) + if err != nil { + return nil, err + } + values, err := redisBucketValues(ctx, keys) + if err != nil { + return nil, err } - if sample.LatencyMs > 0 { - pipe.HIncrBy(ctx, redisKey, "lat", sample.LatencyMs) + type decodedBucket struct { + key bucketKey + value counters } - if sample.HasTtft && sample.TtftMs >= 0 { - pipe.HIncrBy(ctx, redisKey, "ttft", sample.TtftMs) - pipe.HIncrBy(ctx, redisKey, "ttft_n", 1) + decoded := make([]decodedBucket, 0, len(values)) + receiptKeys := make([]string, 0, len(values)) + for _, fields := range values { + key, value, ok := decodeRedisBucket(fields) + if !ok || !matchesActiveBucket(key, filter) { + continue + } + if _, suppressed := committedBuckets.Load(key); suppressed { + continue + } + decoded = append(decoded, decodedBucket{key: key, value: value}) + receiptKeys = append(receiptKeys, perfMetricReceiptKey(key)) } - if sample.OutputTokens > 0 && sample.GenerationMs > 0 { - pipe.HIncrBy(ctx, redisKey, "out", sample.OutputTokens) - pipe.HIncrBy(ctx, redisKey, "gen_ms", sample.GenerationMs) + receipts, err := model.GetPerfMetricFlushReceiptKeys(receiptKeys) + if err != nil { + return nil, err } - pipe.Expire(ctx, redisKey, time.Hour) - _, _ = pipe.Exec(ctx) + merged := make(map[bucketKey]counters, len(decoded)) + for _, bucket := range decoded { + if _, committed := receipts[perfMetricReceiptKey(bucket.key)]; committed { + continue + } + mergeCounters(merged, bucket.key, bucket.value) + } + return merged, nil } -func mergeRedisActiveBuckets(merged map[bucketKey]counters, params QueryParams, startTs int64, endTs int64) { - if !common.RedisEnabled || common.RDB == nil || params.Model == "" || params.Group == "" { +func redisBucketKey(key bucketKey) string { + identity := sha256.Sum256([]byte(fmt.Sprintf( + "%s\x00%s\x00%s\x00%d", + key.node, + key.model, + key.group, + key.bucketSeconds, + ))) + return fmt.Sprintf("perf:v2:bucket:%x:%d", identity[:12], key.bucketTs) +} + +func redisNodeBucketIndex(node string) string { + identity := sha256.Sum256([]byte(node)) + return fmt.Sprintf("perf:v2:node:%x", identity[:12]) +} + +func redisModelBucketIndex(modelName string) string { + identity := sha256.Sum256([]byte(modelName)) + return fmt.Sprintf("perf:v2:model:%x", identity[:12]) +} + +func redisProjectionHealthKey(node string) string { + identity := sha256.Sum256([]byte(node)) + return fmt.Sprintf("perf:v2:health:%x", identity[:12]) +} + +func setProjectionGap(key bucketKey, incomplete bool) { + if incomplete { + if _, loaded := projectionGapBuckets.LoadOrStore(key, struct{}{}); !loaded { + projectionGapCount.Add(1) + } + degradedUntil := time.Now().Add(redisBucketTTL).Unix() + for { + current := projectionDegradedUntil.Load() + if current >= degradedUntil || projectionDegradedUntil.CompareAndSwap(current, degradedUntil) { + break + } + } return } - active := bucketStart(time.Now().Unix()) - if active < startTs || active > endTs { - return + clearProjectionGap(key) +} + +func clearProjectionGap(key bucketKey) { + if _, loaded := projectionGapBuckets.LoadAndDelete(key); loaded { + if projectionGapCount.Add(-1) == 0 { + projectionDegradedUntil.Store(0) + } + } +} + +func localProjectionDegraded() bool { + return projectionGapCount.Load() > 0 || projectionDegradedUntil.Load() > time.Now().Unix() +} + +func projectionHealthLoop() { + ticker := time.NewTicker(projectionHealthInterval) + defer ticker.Stop() + for range ticker.C { + if err := publishProjectionHealth(); err != nil && common.RedisEnabled { + common.SysError("failed to publish performance projection health: " + err.Error()) + } + } +} + +func publishProjectionHealth() error { + if !common.RedisEnabled || common.RDB == nil { + return nil } - key := bucketKey{model: params.Model, group: params.Group, bucketTs: active} ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - values, err := common.RDB.HGetAll(ctx, redisBucketKey(key)).Result() - if err != nil || len(values) == 0 { - return + degraded := localProjectionDegraded() + ttl := projectionHealthyTTL + if degraded { + ttl = redisBucketTTL } - mergeCounters(merged, key, redisCounters(values)) + expiresAt := time.Now().Add(ttl).Unix() + key := redisProjectionHealthKey(currentNodeName()) + pipe := common.RDB.TxPipeline() + pipe.HSet(ctx, key, map[string]interface{}{ + "node": currentNodeName(), + "degraded": boolToRedisInt(degraded), + "degraded_until": projectionDegradedUntil.Load(), + "updated_at": time.Now().Unix(), + }) + pipe.Expire(ctx, key, ttl) + pipe.ZAdd(ctx, redisProjectionHealthIndex, &redis.Z{Score: float64(expiresAt), Member: key}) + pipe.Expire(ctx, redisProjectionHealthIndex, redisBucketTTL+time.Hour) + _, err := pipe.Exec(ctx) + return err } -func redisBucketKey(key bucketKey) string { - return fmt.Sprintf("perf:%s:%s:%d", key.model, key.group, key.bucketTs) +func restoreProjectionHealth() error { + if !common.RedisEnabled || common.RDB == nil { + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + fields, err := common.RDB.HGetAll(ctx, redisProjectionHealthKey(currentNodeName())).Result() + if err != nil { + return err + } + degradedUntil := parseRedisInt(fields["degraded_until"]) + if fields["degraded"] == "1" && degradedUntil > time.Now().Unix() { + projectionDegradedUntil.Store(degradedUntil) + } + return nil +} + +func resetProjectionHealthForTest() { + projectionGapBuckets = sync.Map{} + projectionGapCount.Store(0) + projectionDegradedUntil.Store(0) +} + +func loadProjectionHealth() (bool, error) { + if !common.RedisEnabled || common.RDB == nil { + return false, nil + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + keys, err := activeRedisKeys(ctx, redisProjectionHealthIndex) + if err != nil { + return false, err + } + values, err := redisBucketValues(ctx, keys) + if err != nil { + return false, err + } + for _, fields := range values { + if fields["degraded"] == "1" { + return true, nil + } + } + return false, nil +} + +func boolToRedisInt(value bool) int { + if value { + return 1 + } + return 0 +} + +func currentNodeName() string { + if common.NodeName != "" { + return common.NodeName + } + return "unnamed" +} + +func activeRedisKeys(ctx context.Context, index string) ([]string, error) { + now := time.Now().Unix() + pipe := common.RDB.TxPipeline() + pipe.ZRemRangeByScore(ctx, index, "-inf", strconv.FormatInt(now, 10)) + keysCmd := pipe.ZRangeByScore(ctx, index, &redis.ZRangeBy{Min: strconv.FormatInt(now+1, 10), Max: "+inf"}) + if _, err := pipe.Exec(ctx); err != nil { + return nil, err + } + return keysCmd.Val(), nil +} + +func redisBucketValues(ctx context.Context, keys []string) ([]map[string]string, error) { + if len(keys) == 0 { + return nil, nil + } + pipe := common.RDB.Pipeline() + commands := make([]*redis.StringStringMapCmd, 0, len(keys)) + for _, key := range keys { + commands = append(commands, pipe.HGetAll(ctx, key)) + } + if _, err := pipe.Exec(ctx); err != nil { + return nil, err + } + values := make([]map[string]string, 0, len(commands)) + for _, command := range commands { + if fields := command.Val(); len(fields) > 0 && fields["flushed"] != "1" { + values = append(values, fields) + } + } + return values, nil +} + +func decodeRedisBucket(fields map[string]string) (bucketKey, counters, bool) { + key := bucketKey{ + model: fields["model"], + group: fields["group"], + node: fields["node"], + bucketTs: parseRedisInt(fields["bucket_ts"]), + bucketSeconds: parseRedisInt(fields["bucket_seconds"]), + } + value := redisCounters(fields) + return key, value, key.model != "" && key.group != "" && key.node != "" && key.bucketTs > 0 && value.requestCount > 0 +} + +func recoverRedisBucketsForNode() (int, error) { + if !common.RedisEnabled || common.RDB == nil { + return 0, nil + } + node := currentNodeName() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + keys, err := activeRedisKeys(ctx, redisNodeBucketIndex(node)) + if err != nil { + return 0, err + } + values, err := redisBucketValues(ctx, keys) + if err != nil { + return 0, err + } + type recoveredBucket struct { + key bucketKey + value counters + } + candidates := make([]recoveredBucket, 0, len(values)) + receiptKeys := make([]string, 0, len(values)) + for _, fields := range values { + key, value, ok := decodeRedisBucket(fields) + if !ok || key.node != node { + continue + } + candidates = append(candidates, recoveredBucket{key: key, value: value}) + receiptKeys = append(receiptKeys, perfMetricReceiptKey(key)) + } + receipts, err := model.GetPerfMetricFlushReceiptKeys(receiptKeys) + if err != nil { + return 0, err + } + recovered := 0 + for _, candidate := range candidates { + if _, committed := receipts[perfMetricReceiptKey(candidate.key)]; committed { + if err := deleteRedisBucket(candidate.key); err != nil { + committedBuckets.Store(candidate.key, struct{}{}) + } + continue + } + bucket := &atomicBucket{} + bucket.addCounters(candidate.value) + hotBuckets.Store(candidate.key, bucket) + recovered++ + } + return recovered, nil } diff --git a/pkg/perf_metrics/metrics_test.go b/pkg/perf_metrics/metrics_test.go new file mode 100644 index 00000000..eecc6e01 --- /dev/null +++ b/pkg/perf_metrics/metrics_test.go @@ -0,0 +1,676 @@ +package perfmetrics + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/MAX-API-Next/MAX-API/common" + "github.com/MAX-API-Next/MAX-API/model" + "github.com/MAX-API-Next/MAX-API/setting/config" + "github.com/MAX-API-Next/MAX-API/setting/perf_metrics_setting" + "github.com/alicebob/miniredis/v2" + "github.com/glebarez/sqlite" + "github.com/go-redis/redis/v8" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +func TestQuerySummaryAllRangeReportsBucketCoverageAndGroupFilter(t *testing.T) { + previousDB := model.DB + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + model.DB = db + hotBuckets = sync.Map{} + t.Cleanup(func() { + model.DB = previousDB + hotBuckets = sync.Map{} + }) + require.NoError(t, db.AutoMigrate(&model.PerfMetric{}, &model.PerfMetricFlushReceipt{})) + require.NoError(t, db.Create(&[]model.PerfMetric{ + {ModelName: "alpha", Group: "prod", BucketTs: 1_000, RequestCount: 2, SuccessCount: 2, TotalLatencyMs: 400, OutputTokens: 100, GenerationMs: 2_000}, + {ModelName: "alpha", Group: "prod", BucketTs: 2_000, RequestCount: 1, SuccessCount: 0, TotalLatencyMs: 300, OutputTokens: 50, GenerationMs: 1_000}, + {ModelName: "alpha", Group: "staging", BucketTs: 1_500, RequestCount: 10, SuccessCount: 10, TotalLatencyMs: 100, OutputTokens: 1_000, GenerationMs: 1_000}, + {ModelName: "beta", Group: "prod", BucketTs: 2_600, RequestCount: 1, SuccessCount: 1, TotalLatencyMs: 50, OutputTokens: 100, GenerationMs: 1_000}, + }).Error) + + result, err := QuerySummaryAllRangeDetailed(900, 2_500, []string{"prod"}) + require.NoError(t, err) + require.Len(t, result.Models, 1) + require.Equal(t, "alpha", result.Models[0].ModelName) + require.EqualValues(t, 3, result.Models[0].RequestCount) + require.EqualValues(t, 233, result.Models[0].AvgLatencyMs) + require.InDelta(t, 66.67, result.Models[0].SuccessRate, 0.001) + require.InDelta(t, 50, result.Models[0].AvgTps, 0.001) + require.Equal(t, CollectionStateAvailable, result.CollectionState) + require.EqualValues(t, 900, result.Coverage.RequestedStartAt) + require.EqualValues(t, 2_500, result.Coverage.RequestedEndAt) + require.EqualValues(t, 1_000, result.Coverage.BucketStartAt) + require.EqualValues(t, 2_500, result.Coverage.BucketEndAt) + require.EqualValues(t, 3_600, result.Coverage.BucketSeconds) + require.True(t, result.Coverage.Approximate) + require.Equal(t, CoverageGranularityUnknown, result.Coverage.GranularityState) +} + +func TestBuildQueryResultUsesRawCountersForModelAggregate(t *testing.T) { + result := buildQueryResult("alpha", map[bucketKey]counters{ + {model: "alpha", group: "small", bucketTs: 3_600}: { + requestCount: 1, + successCount: 0, + totalLatencyMs: 1_000, + ttftSumMs: 100, + ttftCount: 1, + outputTokens: 10, + generationMs: 1_000, + }, + {model: "alpha", group: "large", bucketTs: 3_600}: { + requestCount: 99, + successCount: 99, + totalLatencyMs: 9_900, + ttftSumMs: 4_950, + ttftCount: 99, + outputTokens: 990, + generationMs: 9_900, + }, + }) + + require.Len(t, result.Groups, 2) + require.EqualValues(t, 109, result.Summary.AvgLatencyMs) + require.EqualValues(t, 50, result.Summary.AvgTtftMs) + require.InDelta(t, 99, result.Summary.SuccessRate, 0.001) + require.InDelta(t, 91.743, result.Summary.AvgTps, 0.001) + require.Len(t, result.Summary.Series, 1) + require.InDelta(t, 99, result.Summary.Series[0].SuccessRate, 0.001) +} + +func TestBuildQueryResultMergesSameGroupBucketAcrossNodes(t *testing.T) { + result := buildQueryResult("alpha", map[bucketKey]counters{ + {model: "alpha", group: "prod", node: "node-a", bucketTs: 3_600}: { + requestCount: 2, + successCount: 2, + totalLatencyMs: 200, + ttftSumMs: 80, + ttftCount: 2, + outputTokens: 20, + generationMs: 1_000, + }, + {model: "alpha", group: "prod", node: "node-b", bucketTs: 3_600}: { + requestCount: 3, + successCount: 2, + totalLatencyMs: 600, + ttftSumMs: 120, + ttftCount: 3, + outputTokens: 30, + generationMs: 1_500, + }, + }) + + require.Len(t, result.Groups, 1) + require.EqualValues(t, 160, result.Groups[0].AvgLatencyMs) + require.EqualValues(t, 40, result.Groups[0].AvgTtftMs) + require.InDelta(t, 80, result.Groups[0].SuccessRate, 0.001) + require.InDelta(t, 80, result.Summary.SuccessRate, 0.001) +} + +func TestAtomicBucketCloseAndDrainRejectsLateSample(t *testing.T) { + bucket := &atomicBucket{} + require.True(t, bucket.add(Sample{Success: true, LatencyMs: 100})) + drained := bucket.closeAndDrain() + require.EqualValues(t, 1, drained.requestCount) + require.False(t, bucket.add(Sample{Success: true, LatencyMs: 50})) + require.Zero(t, bucket.snapshot().requestCount) +} + +func TestFlushOnceDrainsAcceptedBucketsWhenCollectionIsDisabled(t *testing.T) { + previousDB := model.DB + previousRDB := common.RDB + previousRedisEnabled := common.RedisEnabled + setting := config.GlobalConfig.Get("perf_metrics_setting").(*perf_metrics_setting.PerfMetricsSetting) + previousSetting := *setting + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&model.PerfMetric{}, &model.PerfMetricFlushReceipt{})) + model.DB = db + common.RDB = nil + common.RedisEnabled = false + *setting = previousSetting + setting.Enabled = false + hotBuckets = sync.Map{} + t.Cleanup(func() { + model.DB = previousDB + common.RDB = previousRDB + common.RedisEnabled = previousRedisEnabled + *setting = previousSetting + hotBuckets = sync.Map{} + }) + + key := bucketKey{ + model: "alpha", + group: "prod", + node: "node-a", + bucketTs: bucketStart(time.Now().Unix()) - 3_600, + bucketSeconds: 3_600, + } + bucket := &atomicBucket{} + require.True(t, bucket.add(Sample{Success: true, LatencyMs: 125})) + hotBuckets.Store(key, bucket) + + flushOnce() + + rows, err := model.GetPerfMetrics(key.model, key.group, key.bucketTs, key.bucketTs) + require.NoError(t, err) + require.Len(t, rows, 1) + require.EqualValues(t, 1, rows[0].RequestCount) +} + +func TestAtomicBucketSnapshotWaitsForLifecycleLock(t *testing.T) { + bucket := &atomicBucket{} + bucket.mu.Lock() + done := make(chan struct{}) + go func() { + _ = bucket.snapshot() + close(done) + }() + + select { + case <-done: + bucket.mu.Unlock() + t.Fatal("snapshot returned while the bucket lifecycle lock was held") + case <-time.After(50 * time.Millisecond): + bucket.mu.Unlock() + } + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("snapshot did not resume after the lifecycle lock was released") + } +} + +func TestAtomicBucketCloseWaitsForInFlightRedisProjection(t *testing.T) { + bucket := &atomicBucket{} + value, version, added := bucket.addForProjection(Sample{Success: true, LatencyMs: 100}) + require.True(t, added) + require.EqualValues(t, 1, value.requestCount) + + drained := make(chan counters, 1) + go func() { + drained <- bucket.closeAndDrain() + }() + select { + case <-drained: + t.Fatal("bucket drained before its in-flight Redis projection completed") + case <-time.After(50 * time.Millisecond): + } + + require.False(t, bucket.finishProjection(version, true)) + select { + case result := <-drained: + require.EqualValues(t, 1, result.requestCount) + require.EqualValues(t, 1, result.successCount) + case <-time.After(time.Second): + t.Fatal("bucket did not drain after the Redis projection completed") + } +} + +func TestRemoveClosedBucketDoesNotDeleteReplacement(t *testing.T) { + hotBuckets = sync.Map{} + key := bucketKey{model: "alpha", group: "prod", node: "node", bucketTs: 3_600, bucketSeconds: 3_600} + closed := &atomicBucket{} + require.True(t, closed.add(Sample{Success: true})) + closed.closeAndDrain() + replacement := &atomicBucket{} + require.True(t, replacement.add(Sample{Success: true})) + hotBuckets.Store(key, replacement) + t.Cleanup(func() { hotBuckets = sync.Map{} }) + + removeClosedBucket(key) + + actual, ok := hotBuckets.Load(key) + require.True(t, ok) + require.Same(t, replacement, actual) +} + +func TestQueryDetailedExcludesInactiveGroupsBeforeAggregate(t *testing.T) { + previousDB := model.DB + previousRDB := common.RDB + previousRedisEnabled := common.RedisEnabled + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&model.PerfMetric{}, &model.PerfMetricFlushReceipt{})) + model.DB = db + common.RDB = nil + common.RedisEnabled = false + hotBuckets = sync.Map{} + t.Cleanup(func() { + model.DB = previousDB + common.RDB = previousRDB + common.RedisEnabled = previousRedisEnabled + hotBuckets = sync.Map{} + }) + + bucketSeconds := int64(3_600) + bucketTs := bucketStartFor(time.Now().Unix(), bucketSeconds) + require.NoError(t, db.Create(&[]model.PerfMetric{ + {ModelName: "alpha", Group: "prod", BucketTs: bucketTs, RequestCount: 99, SuccessCount: 99, TotalLatencyMs: 9_900}, + {ModelName: "alpha", Group: "removed", BucketTs: bucketTs, RequestCount: 1, SuccessCount: 0, TotalLatencyMs: 1_000}, + }).Error) + + result, err := QueryDetailed(QueryParams{ + Model: "alpha", + Hours: 1, + AllowedGroups: []string{"prod"}, + }) + + require.NoError(t, err) + require.Len(t, result.Groups, 1) + require.Equal(t, "prod", result.Groups[0].Group) + require.InDelta(t, 100, result.Summary.SuccessRate, 0.001) + require.EqualValues(t, 100, result.Summary.AvgLatencyMs) +} + +func TestQueryDetailedMarksRedisFailureAsPartialWhenDatabaseHasSamples(t *testing.T) { + previousDB := model.DB + previousRDB := common.RDB + previousRedisEnabled := common.RedisEnabled + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&model.PerfMetric{}, &model.PerfMetricFlushReceipt{})) + redisServer := miniredis.RunT(t) + redisClient := redis.NewClient(&redis.Options{Addr: redisServer.Addr()}) + require.NoError(t, redisClient.Close()) + model.DB = db + common.RDB = redisClient + common.RedisEnabled = true + hotBuckets = sync.Map{} + t.Cleanup(func() { + model.DB = previousDB + common.RDB = previousRDB + common.RedisEnabled = previousRedisEnabled + hotBuckets = sync.Map{} + }) + + bucketTs := bucketStart(time.Now().Unix()) + require.NoError(t, db.Create(&model.PerfMetric{ + ModelName: "alpha", Group: "prod", BucketTs: bucketTs, + RequestCount: 1, SuccessCount: 1, TotalLatencyMs: 100, + }).Error) + + result, err := QueryDetailed(QueryParams{Model: "alpha", Hours: 1}) + + require.NoError(t, err) + require.Equal(t, CollectionStatePartial, result.CollectionState) + require.Len(t, result.Groups, 1) +} + +func TestQueryDetailedMarksRedisFailureAsQueryFailedWithoutSamples(t *testing.T) { + previousDB := model.DB + previousRDB := common.RDB + previousRedisEnabled := common.RedisEnabled + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&model.PerfMetric{}, &model.PerfMetricFlushReceipt{})) + redisServer := miniredis.RunT(t) + redisClient := redis.NewClient(&redis.Options{Addr: redisServer.Addr()}) + require.NoError(t, redisClient.Close()) + model.DB = db + common.RDB = redisClient + common.RedisEnabled = true + hotBuckets = sync.Map{} + t.Cleanup(func() { + model.DB = previousDB + common.RDB = previousRDB + common.RedisEnabled = previousRedisEnabled + hotBuckets = sync.Map{} + }) + + result, err := QueryDetailed(QueryParams{Model: "alpha", Hours: 1}) + + require.NoError(t, err) + require.Equal(t, CollectionStateQueryFailed, result.CollectionState) + require.Empty(t, result.Groups) +} + +func TestQuerySummaryAllRangeDetailedMarksEmptyWindowAsNoSamples(t *testing.T) { + previousDB := model.DB + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + model.DB = db + hotBuckets = sync.Map{} + t.Cleanup(func() { + model.DB = previousDB + hotBuckets = sync.Map{} + }) + require.NoError(t, db.AutoMigrate(&model.PerfMetric{}, &model.PerfMetricFlushReceipt{})) + + result, err := QuerySummaryAllRangeDetailed(900, 2_500, nil) + + require.NoError(t, err) + require.Empty(t, result.Models) + require.Equal(t, CollectionStateNoSamples, result.CollectionState) + require.EqualValues(t, 900, result.Coverage.RequestedStartAt) + require.EqualValues(t, 2_500, result.Coverage.RequestedEndAt) + require.Zero(t, result.Coverage.BucketStartAt) + require.Zero(t, result.Coverage.BucketEndAt) + require.False(t, result.Coverage.Approximate) +} + +func TestResolveCollectionStateDistinguishesNormalOperationalStates(t *testing.T) { + require.Equal(t, CollectionStateDisabled, resolveCollectionState(false, true, activeCollectionHealthy)) + require.Equal(t, CollectionStateDisabled, resolveCollectionState(false, false, activeCollectionHealthy)) + require.Equal(t, CollectionStateNoSamples, resolveCollectionState(true, false, activeCollectionHealthy)) + require.Equal(t, CollectionStateAvailable, resolveCollectionState(true, true, activeCollectionHealthy)) + require.Equal(t, CollectionStatePartial, resolveCollectionState(true, true, activeCollectionFailed)) + require.Equal(t, CollectionStateQueryFailed, resolveCollectionState(true, false, activeCollectionFailed)) +} + +func TestQuerySummaryAllRangeDetailedMergesRedisBucketsAcrossNodesWithoutLocalDoubleCounting(t *testing.T) { + previousDB := model.DB + previousRDB := common.RDB + previousRedisEnabled := common.RedisEnabled + previousNodeName := common.NodeName + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&model.PerfMetric{}, &model.PerfMetricFlushReceipt{})) + redisServer := miniredis.RunT(t) + redisClient := redis.NewClient(&redis.Options{Addr: redisServer.Addr()}) + model.DB = db + common.RDB = redisClient + common.RedisEnabled = true + hotBuckets = sync.Map{} + t.Cleanup(func() { + model.DB = previousDB + common.RDB = previousRDB + common.RedisEnabled = previousRedisEnabled + common.NodeName = previousNodeName + hotBuckets = sync.Map{} + _ = redisClient.Close() + }) + + common.NodeName = "perf-node-a" + Record(Sample{Model: "alpha", Group: "prod", LatencyMs: 100, Success: true}) + common.NodeName = "perf-node-b" + Record(Sample{Model: "alpha", Group: "prod", LatencyMs: 300, Success: false}) + + now := time.Now().Unix() + result, err := QuerySummaryAllRangeDetailed(now-3600, now+1, []string{"prod"}) + + require.NoError(t, err) + require.Len(t, result.Models, 1) + require.EqualValues(t, 2, result.Models[0].RequestCount) + require.EqualValues(t, 200, result.Models[0].AvgLatencyMs) + require.InDelta(t, 50, result.Models[0].SuccessRate, 0.001) +} + +func TestRecoverRedisBucketsForNodeRestoresFlushStateAfterRestart(t *testing.T) { + previousDB := model.DB + previousRDB := common.RDB + previousRedisEnabled := common.RedisEnabled + previousNodeName := common.NodeName + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&model.PerfMetric{}, &model.PerfMetricFlushReceipt{})) + redisServer := miniredis.RunT(t) + redisClient := redis.NewClient(&redis.Options{Addr: redisServer.Addr()}) + model.DB = db + common.RDB = redisClient + common.RedisEnabled = true + common.NodeName = "stable-perf-node" + hotBuckets = sync.Map{} + t.Cleanup(func() { + model.DB = previousDB + common.RDB = previousRDB + common.RedisEnabled = previousRedisEnabled + common.NodeName = previousNodeName + hotBuckets = sync.Map{} + _ = redisClient.Close() + }) + + Record(Sample{Model: "alpha", Group: "prod", LatencyMs: 125, Success: true}) + hotBuckets = sync.Map{} + + recovered, err := recoverRedisBucketsForNode() + + require.NoError(t, err) + require.Equal(t, 1, recovered) + requestCount := int64(0) + hotBuckets.Range(func(_, value any) bool { + requestCount += value.(*atomicBucket).snapshot().requestCount + return true + }) + require.EqualValues(t, 1, requestCount) +} + +func TestRecoveredCompletedRedisBucketFlushesToDatabaseAndCleansActiveState(t *testing.T) { + previousDB := model.DB + previousRDB := common.RDB + previousRedisEnabled := common.RedisEnabled + previousNodeName := common.NodeName + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&model.PerfMetric{}, &model.PerfMetricFlushReceipt{})) + redisServer := miniredis.RunT(t) + redisClient := redis.NewClient(&redis.Options{Addr: redisServer.Addr()}) + model.DB = db + common.RDB = redisClient + common.RedisEnabled = true + common.NodeName = "stable-flush-node" + hotBuckets = sync.Map{} + t.Cleanup(func() { + model.DB = previousDB + common.RDB = previousRDB + common.RedisEnabled = previousRedisEnabled + common.NodeName = previousNodeName + hotBuckets = sync.Map{} + _ = redisClient.Close() + }) + + bucketSeconds := int64(3600) + key := bucketKey{ + model: "alpha", + group: "prod", + node: common.NodeName, + bucketTs: bucketStart(time.Now().Unix()) - bucketSeconds, + bucketSeconds: bucketSeconds, + } + require.True(t, recordRedis(key, counters{ + requestCount: 1, + successCount: 1, + totalLatencyMs: 250, + })) + recovered, err := recoverRedisBucketsForNode() + require.NoError(t, err) + require.Equal(t, 1, recovered) + + flushCompletedBuckets() + + rows, err := model.GetPerfMetrics(key.model, key.group, key.bucketTs, key.bucketTs) + require.NoError(t, err) + require.Len(t, rows, 1) + require.EqualValues(t, 1, rows[0].RequestCount) + require.EqualValues(t, 1, rows[0].SuccessCount) + require.EqualValues(t, 250, rows[0].TotalLatencyMs) + require.EqualValues(t, 0, redisClient.Exists(context.Background(), redisBucketKey(key)).Val()) + _, hot := hotBuckets.Load(key) + require.False(t, hot) +} + +func TestRecoveredCommittedRedisBucketDoesNotIncrementDatabaseTwice(t *testing.T) { + previousDB := model.DB + previousRDB := common.RDB + previousRedisEnabled := common.RedisEnabled + previousNodeName := common.NodeName + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&model.PerfMetric{}, &model.PerfMetricFlushReceipt{})) + redisServer := miniredis.RunT(t) + redisClient := redis.NewClient(&redis.Options{Addr: redisServer.Addr()}) + model.DB = db + common.RDB = redisClient + common.RedisEnabled = true + common.NodeName = "stable-idempotent-node" + hotBuckets = sync.Map{} + committedBuckets = sync.Map{} + resetProjectionHealthForTest() + t.Cleanup(func() { + model.DB = previousDB + common.RDB = previousRDB + common.RedisEnabled = previousRedisEnabled + common.NodeName = previousNodeName + hotBuckets = sync.Map{} + committedBuckets = sync.Map{} + resetProjectionHealthForTest() + _ = redisClient.Close() + }) + + key := bucketKey{ + model: "alpha", + group: "prod", + node: common.NodeName, + bucketTs: bucketStart(time.Now().Unix()) - 3600, + bucketSeconds: 3600, + } + value := counters{requestCount: 1, successCount: 1, totalLatencyMs: 100} + require.True(t, recordRedis(key, value)) + applied, err := persistBucket(key, value) + require.NoError(t, err) + require.True(t, applied) + reapplied, err := persistBucket(key, value) + require.NoError(t, err) + require.False(t, reapplied, "the durable receipt must make a repeated commit a no-op") + + // Simulate termination after the database transaction commits but before + // the Redis bucket and process-local receipt cache are cleaned. + hotBuckets = sync.Map{} + committedBuckets = sync.Map{} + recovered, err := recoverRedisBucketsForNode() + require.NoError(t, err) + require.Zero(t, recovered) + require.EqualValues(t, 0, redisClient.Exists(context.Background(), redisBucketKey(key)).Val()) + + flushCompletedBuckets() + + rows, err := model.GetPerfMetrics(key.model, key.group, key.bucketTs, key.bucketTs) + require.NoError(t, err) + require.Len(t, rows, 1) + require.EqualValues(t, 1, rows[0].RequestCount) + require.EqualValues(t, 1, rows[0].SuccessCount) + require.EqualValues(t, 100, rows[0].TotalLatencyMs) +} + +func TestRedisProjectionGapIsVisibleAcrossNodesAfterRedisRecovers(t *testing.T) { + previousDB := model.DB + previousRDB := common.RDB + previousRedisEnabled := common.RedisEnabled + previousNodeName := common.NodeName + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&model.PerfMetric{}, &model.PerfMetricFlushReceipt{})) + redisServer := miniredis.RunT(t) + redisClient := redis.NewClient(&redis.Options{Addr: redisServer.Addr()}) + model.DB = db + common.RDB = redisClient + common.RedisEnabled = true + common.NodeName = "projection-node-a" + hotBuckets = sync.Map{} + committedBuckets = sync.Map{} + resetProjectionHealthForTest() + t.Cleanup(func() { + model.DB = previousDB + common.RDB = previousRDB + common.RedisEnabled = previousRedisEnabled + common.NodeName = previousNodeName + hotBuckets = sync.Map{} + committedBuckets = sync.Map{} + resetProjectionHealthForTest() + _ = redisClient.Close() + }) + + redisServer.Close() + Record(Sample{Model: "alpha", Group: "prod", Success: true, LatencyMs: 100}) + hotBuckets = sync.Map{} // another process cannot access node A's local bucket + require.NoError(t, redisServer.Restart()) + require.NoError(t, publishProjectionHealth()) + resetProjectionHealthForTest() // simulate the independent node B process + + common.NodeName = "projection-node-b" + Record(Sample{Model: "alpha", Group: "prod", Success: true, LatencyMs: 200}) + result, err := QueryDetailed(QueryParams{Model: "alpha", Hours: 1}) + require.NoError(t, err) + require.Equal(t, CollectionStatePartial, result.CollectionState) +} + +func TestRedisAbsoluteSnapshotIsIdempotentWhenAWriteIsRetried(t *testing.T) { + previousRDB := common.RDB + previousRedisEnabled := common.RedisEnabled + redisServer := miniredis.RunT(t) + redisClient := redis.NewClient(&redis.Options{Addr: redisServer.Addr()}) + common.RDB = redisClient + common.RedisEnabled = true + t.Cleanup(func() { + common.RDB = previousRDB + common.RedisEnabled = previousRedisEnabled + _ = redisClient.Close() + }) + + key := bucketKey{model: "alpha", group: "prod", node: "node-a", bucketTs: bucketStart(time.Now().Unix()), bucketSeconds: 3600} + value := counters{requestCount: 1, successCount: 1, totalLatencyMs: 100} + require.True(t, recordRedis(key, value)) + require.True(t, recordRedis(key, value)) + + fields, err := redisClient.HGetAll(context.Background(), redisBucketKey(key)).Result() + require.NoError(t, err) + require.EqualValues(t, 1, parseRedisInt(fields["req"])) + require.EqualValues(t, 1, parseRedisInt(fields["ok"])) + require.EqualValues(t, 100, parseRedisInt(fields["lat"])) +} + +func TestModelQueryUsesModelScopedRedisIndex(t *testing.T) { + previousDB := model.DB + previousRDB := common.RDB + previousRedisEnabled := common.RedisEnabled + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&model.PerfMetric{}, &model.PerfMetricFlushReceipt{})) + redisServer := miniredis.RunT(t) + redisClient := redis.NewClient(&redis.Options{Addr: redisServer.Addr()}) + model.DB = db + common.RDB = redisClient + common.RedisEnabled = true + t.Cleanup(func() { + model.DB = previousDB + common.RDB = previousRDB + common.RedisEnabled = previousRedisEnabled + _ = redisClient.Close() + }) + + now := time.Now().Unix() + alphaKey := bucketKey{model: "alpha", group: "prod", node: "node-a", bucketTs: bucketStart(now), bucketSeconds: 3600} + betaKey := bucketKey{model: "beta", group: "prod", node: "node-b", bucketTs: bucketStart(now), bucketSeconds: 3600} + require.True(t, recordRedis(alphaKey, counters{requestCount: 1, successCount: 1})) + require.True(t, recordRedis(betaKey, counters{requestCount: 1, successCount: 1})) + require.NoError(t, redisClient.Del(context.Background(), redisActiveBucketIndex).Err()) + + values, err := loadRedisActiveBuckets(activeBucketFilter{ + model: "alpha", + startTs: alphaKey.bucketTs, + endTs: now, + }) + require.NoError(t, err) + require.Len(t, values, 1) + require.Contains(t, values, alphaKey) +} + +func TestBuildCoverageClampsKnownBucketEndToRequestedEnd(t *testing.T) { + coverage := buildCoverage(2_400, 2_500, map[bucketKey]counters{ + {model: "alpha", group: "prod", bucketTs: 2_480, bucketSeconds: 60}: { + requestCount: 1, + }, + }, 60) + + require.EqualValues(t, 2_500, coverage.BucketEndAt) + require.EqualValues(t, 60, coverage.BucketSeconds) + require.Equal(t, CoverageGranularityKnown, coverage.GranularityState) + require.True(t, coverage.Approximate) +} diff --git a/pkg/perf_metrics/types.go b/pkg/perf_metrics/types.go index af68eb88..13b905ee 100644 --- a/pkg/perf_metrics/types.go +++ b/pkg/perf_metrics/types.go @@ -1,6 +1,9 @@ package perfmetrics -import "sync/atomic" +import ( + "sync" + "sync/atomic" +) type Store interface { Record(sample Sample) @@ -19,9 +22,10 @@ type Sample struct { } type QueryParams struct { - Model string - Group string - Hours int + Model string + Group string + Hours int + AllowedGroups []string } type BucketPoint struct { @@ -32,6 +36,44 @@ type BucketPoint struct { AvgTps float64 `json:"avg_tps"` } +// AggregateResult is the model-level weighted aggregate. It is derived from +// the stored counters, not by averaging already-aggregated group values. +type AggregateResult struct { + AvgTtftMs int64 `json:"avg_ttft_ms"` + AvgLatencyMs int64 `json:"avg_latency_ms"` + SuccessRate float64 `json:"success_rate"` + AvgTps float64 `json:"avg_tps"` + Series []BucketPoint `json:"series"` +} + +type QueryCoverage struct { + RequestedStartAt int64 `json:"requested_start_at"` + RequestedEndAt int64 `json:"requested_end_at"` + BucketStartAt int64 `json:"bucket_start_at"` + BucketEndAt int64 `json:"bucket_end_at"` + BucketSeconds int64 `json:"bucket_seconds"` + GranularityState CoverageGranularity `json:"granularity_state"` + Approximate bool `json:"approximate"` +} + +type CoverageGranularity string + +const ( + CoverageGranularityKnown CoverageGranularity = "known" + CoverageGranularityUnknown CoverageGranularity = "unknown" + CoverageGranularityMixed CoverageGranularity = "mixed" +) + +type CollectionState string + +const ( + CollectionStateAvailable CollectionState = "available" + CollectionStatePartial CollectionState = "partial" + CollectionStateDisabled CollectionState = "collection_disabled" + CollectionStateNoSamples CollectionState = "no_samples" + CollectionStateQueryFailed CollectionState = "query_failed" +) + type GroupResult struct { Group string `json:"group"` AvgTtftMs int64 `json:"avg_ttft_ms"` @@ -42,9 +84,20 @@ type GroupResult struct { } type QueryResult struct { - ModelName string `json:"model_name"` - SeriesSchema string `json:"series_schema"` - Groups []GroupResult `json:"groups"` + ModelName string `json:"model_name"` + SeriesSchema string `json:"series_schema"` + Summary AggregateResult `json:"summary"` + Groups []GroupResult `json:"groups"` +} + +// DetailedQueryResult is used by administrator-facing Smart Operations +// endpoints. The public pricing projection remains QueryResult-compatible; +// these fields make bucket coverage and collection state explicit for +// operational decisions. +type DetailedQueryResult struct { + QueryResult + Coverage QueryCoverage `json:"coverage"` + CollectionState CollectionState `json:"collection_state"` } type ModelSummary struct { @@ -59,10 +112,18 @@ type SummaryAllResult struct { Models []ModelSummary `json:"models"` } +type DetailedSummaryAllResult struct { + Models []ModelSummary `json:"models"` + Coverage QueryCoverage `json:"coverage"` + CollectionState CollectionState `json:"collection_state"` +} + type bucketKey struct { - model string - group string - bucketTs int64 + model string + group string + node string + bucketTs int64 + bucketSeconds int64 } type counters struct { @@ -76,6 +137,11 @@ type counters struct { } type atomicBucket struct { + mu sync.Mutex + projectionWG sync.WaitGroup + closed bool + version uint64 + projected uint64 requestCount atomic.Int64 successCount atomic.Int64 totalLatencyMs atomic.Int64 @@ -85,7 +151,21 @@ type atomicBucket struct { generationMs atomic.Int64 } -func (b *atomicBucket) add(sample Sample) { +func (b *atomicBucket) add(sample Sample) bool { + _, _, added := b.addSample(sample, false) + return added +} + +func (b *atomicBucket) addForProjection(sample Sample) (counters, uint64, bool) { + return b.addSample(sample, true) +} + +func (b *atomicBucket) addSample(sample Sample, project bool) (counters, uint64, bool) { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return counters{}, 0, false + } b.requestCount.Add(1) if sample.Success { b.successCount.Add(1) @@ -101,9 +181,21 @@ func (b *atomicBucket) add(sample Sample) { b.outputTokens.Add(sample.OutputTokens) b.generationMs.Add(sample.GenerationMs) } + if !project { + return counters{}, 0, true + } + b.version++ + b.projectionWG.Add(1) + return b.snapshotLocked(), b.version, true } func (b *atomicBucket) snapshot() counters { + b.mu.Lock() + defer b.mu.Unlock() + return b.snapshotLocked() +} + +func (b *atomicBucket) snapshotLocked() counters { return counters{ requestCount: b.requestCount.Load(), successCount: b.successCount.Load(), @@ -115,7 +207,32 @@ func (b *atomicBucket) snapshot() counters { } } -func (b *atomicBucket) drain() counters { +func (b *atomicBucket) finishProjection(version uint64, success bool) bool { + b.mu.Lock() + if success && version > b.projected { + b.projected = version + } + incomplete := b.projected < b.version + b.mu.Unlock() + b.projectionWG.Done() + return incomplete +} + +func (b *atomicBucket) closeAndDrain() counters { + b.mu.Lock() + if b.closed { + b.mu.Unlock() + return counters{} + } + b.closed = true + b.mu.Unlock() + + // Every sample that entered this bucket has either completed its shared + // Redis projection or recorded a projection gap before the durable flush + // proceeds. This closes the late-write-after-cleanup race. + b.projectionWG.Wait() + b.mu.Lock() + defer b.mu.Unlock() return counters{ requestCount: b.requestCount.Swap(0), successCount: b.successCount.Swap(0), @@ -127,7 +244,12 @@ func (b *atomicBucket) drain() counters { } } -func (b *atomicBucket) addCounters(c counters) { +func (b *atomicBucket) addCounters(c counters) bool { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return false + } if c.requestCount != 0 { b.requestCount.Add(c.requestCount) } @@ -149,4 +271,5 @@ func (b *atomicBucket) addCounters(c counters) { if c.generationMs != 0 { b.generationMs.Add(c.generationMs) } + return true } diff --git a/router/api-router.go b/router/api-router.go index 1b1a4f0f..b0f762a7 100644 --- a/router/api-router.go +++ b/router/api-router.go @@ -227,6 +227,14 @@ func SetApiRouter(router *gin.Engine) { performanceRoute.GET("/logs", controller.GetLogFiles) performanceRoute.DELETE("/logs", controller.CleanupLogFiles) } + smartOpsRoute := apiRouter.Group("/smart-ops") + smartOpsRoute.Use(middleware.AdminAuth()) + { + smartOpsRoute.GET("/channel-performance", controller.GetChannelPerformance) + smartOpsRoute.GET("/channel-performance/detail", controller.GetChannelPerformanceDetail) + smartOpsRoute.GET("/model-performance", controller.GetModelPerformance) + smartOpsRoute.GET("/model-performance/detail", controller.GetModelPerformanceDetail) + } ratioSyncRoute := apiRouter.Group("/ratio_sync") ratioSyncRoute.Use(middleware.RootAuth()) { diff --git a/service/channel_performance.go b/service/channel_performance.go new file mode 100644 index 00000000..2ddd96a8 --- /dev/null +++ b/service/channel_performance.go @@ -0,0 +1,312 @@ +package service + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/MAX-API-Next/MAX-API/common" + "github.com/MAX-API-Next/MAX-API/constant" + "github.com/MAX-API-Next/MAX-API/model" +) + +var ErrInvalidChannelPerformanceQuery = errors.New("invalid channel performance query") + +const ( + defaultChannelPerformanceHours = 1 + channelPerformanceDetailHours = 24 + maxChannelPerformanceHours = 168 + defaultChannelPerformanceLimit = 100 + maxChannelPerformanceLimit = 200 +) + +type ChannelPerformanceQuery = model.ChannelPerformanceQuery + +type ChannelPerformanceTimeRange struct { + StartAt int64 `json:"start_at"` + EndAt int64 `json:"end_at"` + Hours int `json:"hours"` +} + +type ChannelPerformanceSummaryStats struct { + ChannelCount int `json:"channel_count"` + ObservedCount int64 `json:"observed_count"` + ConsumeLogCount int64 `json:"consume_log_count"` + ErrorLogCount int64 `json:"error_log_count"` + ConsumedQuota int64 `json:"consumed_quota"` + RetryLogCount *int64 `json:"retry_log_count"` + LatencySampleCount int64 `json:"latency_sample_count"` + ObservedSuccessRate *float64 `json:"observed_success_rate"` + AvgLoggedLatencyMs *float64 `json:"avg_logged_latency_ms"` + LastObservedAt int64 `json:"last_observed_at"` +} + +type ChannelPerformanceItem struct { + ChannelID int `json:"channel_id"` + ChannelName string `json:"channel_name"` + ChannelType *int `json:"channel_type"` + ChannelStatus *int `json:"channel_status"` + ModelName string `json:"model_name"` + EffectiveGroup string `json:"effective_group"` + ObservedCount int64 `json:"observed_count"` + ConsumeLogCount int64 `json:"consume_log_count"` + ErrorLogCount int64 `json:"error_log_count"` + ConsumedQuota int64 `json:"consumed_quota"` + RetryLogCount *int64 `json:"retry_log_count"` + LatencySampleCount int64 `json:"latency_sample_count"` + ObservedSuccessRate *float64 `json:"observed_success_rate"` + AvgLoggedLatencyMs *float64 `json:"avg_logged_latency_ms"` + LastObservedAt int64 `json:"last_observed_at"` + ProbeLatencyMs *int `json:"probe_latency_ms"` + ProbeTestTime *int64 `json:"probe_test_time"` + QualityFlags []string `json:"quality_flags"` +} + +type ChannelPerformanceSummary struct { + StorageMode string `json:"storage_mode"` + Source string `json:"source"` + Partial bool `json:"partial"` + QualityFlags []string `json:"quality_flags"` + TimeRange ChannelPerformanceTimeRange `json:"time_range"` + Summary ChannelPerformanceSummaryStats `json:"summary"` + Items []ChannelPerformanceItem `json:"items"` + Truncated bool `json:"truncated"` + GeneratedAt int64 `json:"generated_at"` +} + +// ChannelPerformanceReader is deliberately read-only and narrow. Future +// Attempt/Evidence implementations can replace the legacy source without +// changing the controller or the Smart Operations Center UI contract. +type ChannelPerformanceReader interface { + Summary(context.Context, ChannelPerformanceQuery) (ChannelPerformanceSummary, error) +} + +type legacyChannelPerformanceReader struct{} + +func (legacyChannelPerformanceReader) Summary(ctx context.Context, query ChannelPerformanceQuery) (ChannelPerformanceSummary, error) { + query, rangeWasClamped, err := normalizeChannelPerformanceQuery(query) + if err != nil { + return ChannelPerformanceSummary{}, err + } + + legacyResult, err := model.QueryLegacyChannelPerformance(ctx, query) + if err != nil { + return ChannelPerformanceSummary{}, err + } + rows := legacyResult.Rows + limit := query.Limit + truncated := len(rows) > limit + if truncated { + rows = rows[:limit] + } + + ids := make([]int, 0, len(rows)) + seen := make(map[int]struct{}, len(rows)) + for _, row := range rows { + if _, ok := seen[row.ChannelID]; ok { + continue + } + seen[row.ChannelID] = struct{}{} + ids = append(ids, row.ChannelID) + } + channels, err := model.GetChannelPerformanceChannels(ctx, ids) + if err != nil { + return ChannelPerformanceSummary{}, err + } + return buildLegacyChannelPerformanceSummary( + query, + rangeWasClamped, + legacyResult, + rows, + channels, + truncated, + common.LogConsumeEnabled, + constant.ErrorLogEnabled, + time.Now().Unix(), + ), nil +} + +func buildLegacyChannelPerformanceSummary( + query ChannelPerformanceQuery, + rangeWasClamped bool, + legacyResult model.LegacyChannelPerformanceResult, + rows []model.LegacyChannelPerformanceRow, + channels map[int]model.ChannelPerformanceChannel, + truncated bool, + consumeLogsEnabled bool, + errorLogsEnabled bool, + generatedAt int64, +) ChannelPerformanceSummary { + qualityFlags := []string{ + "legacy", + "partial", + "weak_correlation", + "heuristic_outcome", + "coarse_latency", + "probe_separate", + "non_attempt_logs_excluded", + "incomplete_errors_possible", + } + if rangeWasClamped { + qualityFlags = append(qualityFlags, "time_range_clamped") + } + if truncated { + qualityFlags = append(qualityFlags, "truncated") + } + switch legacyResult.RetryMetricState { + case model.LegacyRetryMetricStatePending: + qualityFlags = append(qualityFlags, "retry_backfill_pending") + case model.LegacyRetryMetricStateUnavailable: + qualityFlags = append(qualityFlags, "retry_metrics_unavailable") + } + if !consumeLogsEnabled { + qualityFlags = append(qualityFlags, "consume_logs_disabled") + } + if !errorLogsEnabled { + qualityFlags = append(qualityFlags, "error_logs_disabled") + } + + allowSuccessRate := consumeLogsEnabled && errorLogsEnabled + items := make([]ChannelPerformanceItem, 0, len(rows)) + for _, row := range rows { + metadata, metadataFound := channels[row.ChannelID] + itemFlags := append([]string(nil), qualityFlags...) + item := ChannelPerformanceItem{ + ChannelID: row.ChannelID, + ChannelName: fmt.Sprintf("Channel #%d", row.ChannelID), + ModelName: row.ModelName, + EffectiveGroup: row.EffectiveGroup, + ObservedCount: row.ObservedCount, + ConsumeLogCount: row.ConsumeLogCount, + ErrorLogCount: row.ErrorLogCount, + ConsumedQuota: row.ConsumedQuota, + LatencySampleCount: row.LatencySampleCount, + LastObservedAt: row.LastObservedAt, + QualityFlags: itemFlags, + } + if legacyResult.RetryMetricsAvailable { + item.RetryLogCount = int64Pointer(row.RetryLogCount) + } + if metadataFound { + item.ChannelName = metadata.Name + item.ChannelType = intPointer(metadata.Type) + item.ChannelStatus = intPointer(metadata.Status) + if metadata.ResponseTime > 0 { + item.ProbeLatencyMs = intPointer(metadata.ResponseTime) + } + if metadata.TestTime > 0 { + item.ProbeTestTime = int64Pointer(metadata.TestTime) + } + } else { + item.QualityFlags = append(item.QualityFlags, "metadata_missing") + } + if item.ProbeLatencyMs == nil { + item.QualityFlags = append(item.QualityFlags, "probe_unavailable") + } + if item.LatencySampleCount > 0 { + item.AvgLoggedLatencyMs = floatPointer(float64(row.TotalUseTimeSeconds*1000) / float64(item.LatencySampleCount)) + } + if allowSuccessRate && item.ObservedCount > 0 { + item.ObservedSuccessRate = floatPointer(float64(item.ConsumeLogCount) / float64(item.ObservedCount) * 100) + } + items = append(items, item) + } + + summary := ChannelPerformanceSummaryStats{ + ChannelCount: int(legacyResult.Summary.ChannelCount), + ObservedCount: legacyResult.Summary.ObservedCount, + ConsumeLogCount: legacyResult.Summary.ConsumeLogCount, + ErrorLogCount: legacyResult.Summary.ErrorLogCount, + ConsumedQuota: legacyResult.Summary.ConsumedQuota, + LatencySampleCount: legacyResult.Summary.LatencySampleCount, + LastObservedAt: legacyResult.Summary.LastObservedAt, + } + if legacyResult.RetryMetricsAvailable { + summary.RetryLogCount = int64Pointer(legacyResult.Summary.RetryLogCount) + } + if allowSuccessRate && summary.ObservedCount > 0 { + summary.ObservedSuccessRate = floatPointer(float64(summary.ConsumeLogCount) / float64(summary.ObservedCount) * 100) + } + if summary.LatencySampleCount > 0 { + summary.AvgLoggedLatencyMs = floatPointer(float64(legacyResult.Summary.TotalUseTimeSeconds*1000) / float64(summary.LatencySampleCount)) + } + + return ChannelPerformanceSummary{ + StorageMode: "legacy_log", + Source: "log_db", + Partial: true, + QualityFlags: qualityFlags, + TimeRange: ChannelPerformanceTimeRange{ + StartAt: query.StartAt, + EndAt: query.EndAt, + Hours: int((query.EndAt - query.StartAt) / int64(time.Hour/time.Second)), + }, + Summary: summary, + Items: items, + Truncated: truncated, + GeneratedAt: generatedAt, + } +} + +var defaultChannelPerformanceReader ChannelPerformanceReader = legacyChannelPerformanceReader{} + +func GetChannelPerformance(ctx context.Context, query ChannelPerformanceQuery) (ChannelPerformanceSummary, error) { + return defaultChannelPerformanceReader.Summary(ctx, query) +} + +// GetChannelPerformanceDetail returns a channel-scoped, read-only 24-hour +// log projection. Keeping this contract separate from the list filters avoids +// accidental automatic refreshes and lets a future Attempt-backed reader +// replace the legacy source without changing the administrator API. +func GetChannelPerformanceDetail(ctx context.Context, channelID int) (ChannelPerformanceSummary, error) { + if channelID <= 0 { + return ChannelPerformanceSummary{}, fmt.Errorf("%w: channel_id must be greater than zero", ErrInvalidChannelPerformanceQuery) + } + return defaultChannelPerformanceReader.Summary(ctx, ChannelPerformanceQuery{ + ChannelID: channelID, + Hours: channelPerformanceDetailHours, + Limit: maxChannelPerformanceLimit, + }) +} + +func normalizeChannelPerformanceQuery(query ChannelPerformanceQuery) (ChannelPerformanceQuery, bool, error) { + now := time.Now().Unix() + rangeWasClamped := false + if query.EndAt <= 0 { + query.EndAt = now + } + if query.StartAt <= 0 { + hours := query.Hours + if hours <= 0 { + hours = defaultChannelPerformanceHours + } + if hours > maxChannelPerformanceHours { + hours = maxChannelPerformanceHours + rangeWasClamped = true + } + query.Hours = hours + query.StartAt = query.EndAt - int64(hours)*int64(time.Hour/time.Second) + } + if query.EndAt <= query.StartAt { + return query, false, fmt.Errorf("%w: end must be greater than start", ErrInvalidChannelPerformanceQuery) + } + if query.Limit <= 0 { + query.Limit = defaultChannelPerformanceLimit + } + if query.Limit > maxChannelPerformanceLimit { + query.Limit = maxChannelPerformanceLimit + } + maxWindow := int64(maxChannelPerformanceHours) * int64(time.Hour/time.Second) + if query.EndAt-query.StartAt > maxWindow { + query.StartAt = query.EndAt - maxWindow + rangeWasClamped = true + } + return query, rangeWasClamped, nil +} + +func intPointer(value int) *int { return &value } + +func int64Pointer(value int64) *int64 { return &value } + +func floatPointer(value float64) *float64 { return &value } diff --git a/service/channel_performance_test.go b/service/channel_performance_test.go new file mode 100644 index 00000000..b9d66071 --- /dev/null +++ b/service/channel_performance_test.go @@ -0,0 +1,185 @@ +package service + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/MAX-API-Next/MAX-API/model" + "github.com/stretchr/testify/require" +) + +type capturingChannelPerformanceReader struct { + query ChannelPerformanceQuery +} + +func (reader *capturingChannelPerformanceReader) Summary(_ context.Context, query ChannelPerformanceQuery) (ChannelPerformanceSummary, error) { + reader.query = query + return ChannelPerformanceSummary{TimeRange: ChannelPerformanceTimeRange{Hours: query.Hours}}, nil +} + +func TestNormalizeChannelPerformanceQueryBoundsWindowAndLimit(t *testing.T) { + end := int64(1_700_000_000) + query, clamped, err := normalizeChannelPerformanceQuery(ChannelPerformanceQuery{ + StartAt: end - int64(30*24*time.Hour/time.Second), + EndAt: end, + Limit: 999, + }) + + require.NoError(t, err) + require.True(t, clamped) + require.Equal(t, maxChannelPerformanceLimit, query.Limit) + require.EqualValues(t, maxChannelPerformanceHours, (query.EndAt-query.StartAt)/int64(time.Hour/time.Second)) +} + +func TestNormalizeChannelPerformanceQueryUsesServerOwnedHourWindow(t *testing.T) { + before := time.Now().Unix() + query, clamped, err := normalizeChannelPerformanceQuery(ChannelPerformanceQuery{}) + after := time.Now().Unix() + + require.NoError(t, err) + require.False(t, clamped) + require.GreaterOrEqual(t, query.EndAt, before) + require.LessOrEqual(t, query.EndAt, after) + require.EqualValues(t, time.Hour/time.Second, query.EndAt-query.StartAt) + + query, clamped, err = normalizeChannelPerformanceQuery(ChannelPerformanceQuery{Hours: 37}) + require.NoError(t, err) + require.False(t, clamped) + require.EqualValues(t, 37*time.Hour/time.Second, query.EndAt-query.StartAt) + + query, clamped, err = normalizeChannelPerformanceQuery(ChannelPerformanceQuery{Hours: 999}) + require.NoError(t, err) + require.True(t, clamped) + require.EqualValues(t, maxChannelPerformanceHours*time.Hour/time.Second, query.EndAt-query.StartAt) +} + +func TestNormalizeChannelPerformanceQueryRejectsReversedRange(t *testing.T) { + _, _, err := normalizeChannelPerformanceQuery(ChannelPerformanceQuery{ + StartAt: 20, + EndAt: 10, + }) + + require.Error(t, err) + require.True(t, errors.Is(err, ErrInvalidChannelPerformanceQuery)) +} + +func TestGetChannelPerformanceDetailUsesFixedChannelWindow(t *testing.T) { + previousReader := defaultChannelPerformanceReader + reader := &capturingChannelPerformanceReader{} + defaultChannelPerformanceReader = reader + t.Cleanup(func() { defaultChannelPerformanceReader = previousReader }) + + result, err := GetChannelPerformanceDetail(context.Background(), 17) + + require.NoError(t, err) + require.Equal(t, 17, reader.query.ChannelID) + require.Equal(t, channelPerformanceDetailHours, reader.query.Hours) + require.Equal(t, maxChannelPerformanceLimit, reader.query.Limit) + require.Equal(t, channelPerformanceDetailHours, result.TimeRange.Hours) +} + +func TestGetChannelPerformanceDetailRejectsInvalidChannel(t *testing.T) { + _, err := GetChannelPerformanceDetail(context.Background(), 0) + + require.Error(t, err) + require.ErrorIs(t, err, ErrInvalidChannelPerformanceQuery) +} + +func TestBuildLegacyChannelPerformanceSummaryPreservesFullSummaryContract(t *testing.T) { + retryCount := int64(2) + result := buildLegacyChannelPerformanceSummary( + ChannelPerformanceQuery{StartAt: 1_700_000_000, EndAt: 1_700_003_600, Limit: 1}, + false, + model.LegacyChannelPerformanceResult{ + Rows: []model.LegacyChannelPerformanceRow{ + {ChannelID: 7, ModelName: "alpha", EffectiveGroup: "prod"}, + {ChannelID: 8, ModelName: "beta", EffectiveGroup: "prod"}, + }, + Summary: model.LegacyChannelPerformanceSummary{ + ChannelCount: 2, + ObservedCount: 5, + ConsumeLogCount: 3, + ErrorLogCount: 2, + ConsumedQuota: 900, + RetryLogCount: retryCount, + LatencySampleCount: 4, + TotalUseTimeSeconds: 3, + LastObservedAt: 1_700_003_500, + }, + RetryMetricsAvailable: true, + RetryMetricState: model.LegacyRetryMetricStateAvailable, + }, + []model.LegacyChannelPerformanceRow{ + { + ChannelID: 7, + ModelName: "alpha", + EffectiveGroup: "prod", + ObservedCount: 3, + ConsumeLogCount: 2, + ErrorLogCount: 1, + ConsumedQuota: 500, + RetryLogCount: 1, + LatencySampleCount: 2, + TotalUseTimeSeconds: 1, + LastObservedAt: 1_700_003_400, + }, + }, + map[int]model.ChannelPerformanceChannel{ + 7: {ID: 7, Name: "Primary", Type: 1, Status: 1}, + }, + true, + true, + true, + 1_700_003_600, + ) + + require.True(t, result.Truncated) + require.Len(t, result.Items, 1) + require.Contains(t, result.QualityFlags, "truncated") + require.Equal(t, 2, result.Summary.ChannelCount, "summary must not be reduced to the limited row set") + require.EqualValues(t, 5, result.Summary.ObservedCount) + require.EqualValues(t, 3, result.Summary.ConsumeLogCount) + require.EqualValues(t, 900, result.Summary.ConsumedQuota) + require.NotNil(t, result.Summary.RetryLogCount) + require.EqualValues(t, retryCount, *result.Summary.RetryLogCount) + require.NotNil(t, result.Summary.ObservedSuccessRate) + require.InDelta(t, 60, *result.Summary.ObservedSuccessRate, 0.001) + require.NotNil(t, result.Summary.AvgLoggedLatencyMs) + require.InDelta(t, 750, *result.Summary.AvgLoggedLatencyMs, 0.001, "latency must use the explicit sample count") + require.EqualValues(t, 1_700_003_500, result.Summary.LastObservedAt) + require.EqualValues(t, 1_700_003_600, result.GeneratedAt) + + item := result.Items[0] + require.Equal(t, "Primary", item.ChannelName) + require.NotNil(t, item.ObservedSuccessRate) + require.EqualValues(t, 500, item.ConsumedQuota) + require.InDelta(t, 66.666, *item.ObservedSuccessRate, 0.01) + require.NotNil(t, item.AvgLoggedLatencyMs) + require.InDelta(t, 500, *item.AvgLoggedLatencyMs, 0.001) +} + +func TestBuildLegacyChannelPerformanceSummaryDoesNotInventRetryMetrics(t *testing.T) { + result := buildLegacyChannelPerformanceSummary( + ChannelPerformanceQuery{StartAt: 1, EndAt: 3_601, Limit: 10}, + false, + model.LegacyChannelPerformanceResult{ + Summary: model.LegacyChannelPerformanceSummary{ObservedCount: 1, ConsumeLogCount: 1}, + RetryMetricState: model.LegacyRetryMetricStatePending, + }, + []model.LegacyChannelPerformanceRow{{ + ChannelID: 1, ModelName: "alpha", ObservedCount: 1, ConsumeLogCount: 1, + }}, + nil, + false, + true, + true, + 3_601, + ) + + require.Nil(t, result.Summary.RetryLogCount) + require.Nil(t, result.Items[0].RetryLogCount) + require.Contains(t, result.QualityFlags, "retry_backfill_pending") + require.Contains(t, result.Items[0].QualityFlags, "retry_backfill_pending") +} diff --git a/service/channel_select.go b/service/channel_select.go index 2882290e..87bf85ec 100644 --- a/service/channel_select.go +++ b/service/channel_select.go @@ -21,6 +21,18 @@ type RetryParam struct { excludedChannelIDs map[int]struct{} } +// SetSelectedRoutingGroup keeps the configured route and the actual group in +// separate context keys. ContextKeyUsingGroup is the effective group used by +// logging/billing; ContextKeyTokenGroup and the route plan retain the token's +// configured routing policy. +func SetSelectedRoutingGroup(c *gin.Context, group string) { + if c == nil || group == "" { + return + } + common.SetContextKey(c, constant.ContextKeyAutoGroup, group) + common.SetContextKey(c, constant.ContextKeyUsingGroup, group) +} + func (p *RetryParam) GetRetry() int { if p.Retry == nil { return 0 @@ -148,7 +160,7 @@ func CacheGetRandomSatisfiedChannel(param *RetryParam) (*model.Channel, string, param.SetRetry(0) continue } - common.SetContextKey(param.Ctx, constant.ContextKeyAutoGroup, autoGroup) + SetSelectedRoutingGroup(param.Ctx, autoGroup) selectGroup = autoGroup logger.LogDebug(param.Ctx, "Auto selected group: %s", autoGroup) @@ -229,7 +241,7 @@ func selectChannelFromTokenRoutePlan(param *RetryParam, plan *TokenRoutePlan) (* continue } - common.SetContextKey(param.Ctx, constant.ContextKeyAutoGroup, group) + SetSelectedRoutingGroup(param.Ctx, group) if plan.RetryOnFailure && index < len(plan.OrderedGroups)-1 { common.SetContextKey(param.Ctx, constant.ContextKeyAutoGroupIndex, index+1) common.SetContextKey(param.Ctx, constant.ContextKeyAutoGroupRetryIndex, param.GetRetry()+1) diff --git a/service/channel_select_routing_test.go b/service/channel_select_routing_test.go index fb15fa74..25f3fe37 100644 --- a/service/channel_select_routing_test.go +++ b/service/channel_select_routing_test.go @@ -72,6 +72,7 @@ func TestTokenRoutePlanAdvancesGroupsWithoutResettingRetryBudget(t *testing.T) { require.NotNil(t, first) require.Equal(t, "base", firstGroup) require.Equal(t, 0, param.GetRetry()) + require.Equal(t, "base", common.GetContextKeyString(ctx, constant.ContextKeyUsingGroup)) param.ExcludeChannel(first.Id) param.SetRetry(1) @@ -80,6 +81,7 @@ func TestTokenRoutePlanAdvancesGroupsWithoutResettingRetryBudget(t *testing.T) { require.NotNil(t, second) require.Equal(t, "deluxe", secondGroup) require.Equal(t, 1, param.GetRetry()) + require.Equal(t, "deluxe", common.GetContextKeyString(ctx, constant.ContextKeyUsingGroup)) } func TestTokenRoutePlanDoesNotAdvanceAfterFailureWhenRetryDisabled(t *testing.T) { diff --git a/service/model_performance.go b/service/model_performance.go new file mode 100644 index 00000000..2fa7abe3 --- /dev/null +++ b/service/model_performance.go @@ -0,0 +1,312 @@ +package service + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/MAX-API-Next/MAX-API/common" + "github.com/MAX-API-Next/MAX-API/constant" + "github.com/MAX-API-Next/MAX-API/model" + perfmetrics "github.com/MAX-API-Next/MAX-API/pkg/perf_metrics" +) + +var ErrInvalidModelPerformanceQuery = errors.New("invalid model performance query") + +const modelPerformanceDetailHours = 24 + +type ModelPerformanceQuery = model.ModelPerformanceQuery + +type ModelPerformanceTimeRange struct { + StartAt int64 `json:"start_at"` + EndAt int64 `json:"end_at"` + Hours int `json:"hours"` +} + +type ModelPerformanceSummaryStats struct { + ModelCount int `json:"model_count"` + ChannelCount int `json:"channel_count"` + ObservedCount int64 `json:"observed_count"` + ConsumeLogCount int64 `json:"consume_log_count"` + ErrorLogCount int64 `json:"error_log_count"` + ConsumedQuota int64 `json:"consumed_quota"` + RetryLogCount *int64 `json:"retry_log_count"` + LatencySampleCount int64 `json:"latency_sample_count"` + ObservedSuccessRate *float64 `json:"observed_success_rate"` + AvgLoggedLatencyMs *float64 `json:"avg_logged_latency_ms"` + LastObservedAt int64 `json:"last_observed_at"` +} + +type ModelPerformanceItem struct { + ModelName string `json:"model_name"` + ChannelCount int `json:"channel_count"` + ObservedCount int64 `json:"observed_count"` + ConsumeLogCount int64 `json:"consume_log_count"` + ErrorLogCount int64 `json:"error_log_count"` + ConsumedQuota int64 `json:"consumed_quota"` + RetryLogCount *int64 `json:"retry_log_count"` + LatencySampleCount int64 `json:"latency_sample_count"` + ObservedSuccessRate *float64 `json:"observed_success_rate"` + AvgLoggedLatencyMs *float64 `json:"avg_logged_latency_ms"` + AvgTps *float64 `json:"avg_tps"` + LastObservedAt int64 `json:"last_observed_at"` + QualityFlags []string `json:"quality_flags"` +} + +type ModelPerformanceSummary struct { + StorageMode string `json:"storage_mode"` + Source string `json:"source"` + Partial bool `json:"partial"` + QualityFlags []string `json:"quality_flags"` + TimeRange ModelPerformanceTimeRange `json:"time_range"` + Summary ModelPerformanceSummaryStats `json:"summary"` + Items []ModelPerformanceItem `json:"items"` + Throughput ModelPerformanceThroughputInfo `json:"throughput"` + Truncated bool `json:"truncated"` + GeneratedAt int64 `json:"generated_at"` +} + +type ModelPerformanceReader interface { + Summary(context.Context, ModelPerformanceQuery) (ModelPerformanceSummary, error) +} + +type legacyModelPerformanceReader struct{} + +type modelPerformanceThroughputSnapshot struct { + State perfmetrics.CollectionState + Coverage perfmetrics.QueryCoverage + ByModel map[string]float64 +} + +type ModelPerformanceThroughputInfo struct { + CollectionState perfmetrics.CollectionState `json:"collection_state"` + Coverage perfmetrics.QueryCoverage `json:"coverage"` +} + +func (legacyModelPerformanceReader) Summary(ctx context.Context, query ModelPerformanceQuery) (ModelPerformanceSummary, error) { + query, rangeWasClamped, err := normalizeModelPerformanceQuery(query) + if err != nil { + return ModelPerformanceSummary{}, err + } + + legacyResult, err := model.QueryLegacyModelPerformance(ctx, query) + if err != nil { + return ModelPerformanceSummary{}, err + } + rows := legacyResult.Rows + truncated := len(rows) > query.Limit + if truncated { + rows = rows[:query.Limit] + } + throughput := queryModelPerformanceThroughput(query) + + return buildLegacyModelPerformanceSummary( + query, + rangeWasClamped, + legacyResult, + rows, + truncated, + common.LogConsumeEnabled, + constant.ErrorLogEnabled, + throughput, + time.Now().Unix(), + ), nil +} + +func buildLegacyModelPerformanceSummary( + query ModelPerformanceQuery, + rangeWasClamped bool, + legacyResult model.LegacyModelPerformanceResult, + rows []model.LegacyModelPerformanceRow, + truncated bool, + consumeLogsEnabled bool, + errorLogsEnabled bool, + throughput modelPerformanceThroughputSnapshot, + generatedAt int64, +) ModelPerformanceSummary { + qualityFlags := []string{ + "legacy", + "partial", + "weak_correlation", + "heuristic_outcome", + "coarse_latency", + "non_attempt_logs_excluded", + "incomplete_errors_possible", + } + if rangeWasClamped { + qualityFlags = append(qualityFlags, "time_range_clamped") + } + if truncated { + qualityFlags = append(qualityFlags, "truncated") + } + switch legacyResult.RetryMetricState { + case model.LegacyRetryMetricStatePending: + qualityFlags = append(qualityFlags, "retry_backfill_pending") + case model.LegacyRetryMetricStateUnavailable: + qualityFlags = append(qualityFlags, "retry_metrics_unavailable") + } + if !consumeLogsEnabled { + qualityFlags = append(qualityFlags, "consume_logs_disabled") + } + if !errorLogsEnabled { + qualityFlags = append(qualityFlags, "error_logs_disabled") + } + switch throughput.State { + case perfmetrics.CollectionStatePartial: + qualityFlags = append(qualityFlags, "throughput_partial") + case perfmetrics.CollectionStateDisabled: + qualityFlags = append(qualityFlags, "throughput_collection_disabled") + case perfmetrics.CollectionStateNoSamples: + qualityFlags = append(qualityFlags, "throughput_no_samples") + case perfmetrics.CollectionStateQueryFailed: + qualityFlags = append(qualityFlags, "throughput_query_failed") + } + if throughput.Coverage.Approximate && throughput.State != perfmetrics.CollectionStateNoSamples && throughput.State != perfmetrics.CollectionStateQueryFailed { + qualityFlags = append(qualityFlags, "throughput_window_approximate") + } + + allowSuccessRate := consumeLogsEnabled && errorLogsEnabled + items := make([]ModelPerformanceItem, 0, len(rows)) + for _, row := range rows { + item := ModelPerformanceItem{ + ModelName: row.ModelName, + ChannelCount: int(row.ChannelCount), + ObservedCount: row.ObservedCount, + ConsumeLogCount: row.ConsumeLogCount, + ErrorLogCount: row.ErrorLogCount, + ConsumedQuota: row.ConsumedQuota, + LatencySampleCount: row.LatencySampleCount, + LastObservedAt: row.LastObservedAt, + QualityFlags: append([]string(nil), qualityFlags...), + } + if legacyResult.RetryMetricsAvailable { + item.RetryLogCount = int64Pointer(row.RetryLogCount) + } + if item.LatencySampleCount > 0 { + item.AvgLoggedLatencyMs = floatPointer(float64(row.TotalUseTimeSeconds*1000) / float64(item.LatencySampleCount)) + } + if avgTps, ok := throughput.ByModel[row.ModelName]; ok { + item.AvgTps = floatPointer(avgTps) + } + if allowSuccessRate && item.ObservedCount > 0 { + item.ObservedSuccessRate = floatPointer(float64(item.ConsumeLogCount) / float64(item.ObservedCount) * 100) + } + items = append(items, item) + } + + summary := ModelPerformanceSummaryStats{ + ModelCount: int(legacyResult.Summary.ModelCount), + ChannelCount: int(legacyResult.Summary.ChannelCount), + ObservedCount: legacyResult.Summary.ObservedCount, + ConsumeLogCount: legacyResult.Summary.ConsumeLogCount, + ErrorLogCount: legacyResult.Summary.ErrorLogCount, + ConsumedQuota: legacyResult.Summary.ConsumedQuota, + LatencySampleCount: legacyResult.Summary.LatencySampleCount, + LastObservedAt: legacyResult.Summary.LastObservedAt, + } + if legacyResult.RetryMetricsAvailable { + summary.RetryLogCount = int64Pointer(legacyResult.Summary.RetryLogCount) + } + if allowSuccessRate && summary.ObservedCount > 0 { + summary.ObservedSuccessRate = floatPointer(float64(summary.ConsumeLogCount) / float64(summary.ObservedCount) * 100) + } + if summary.LatencySampleCount > 0 { + summary.AvgLoggedLatencyMs = floatPointer(float64(legacyResult.Summary.TotalUseTimeSeconds*1000) / float64(summary.LatencySampleCount)) + } + + return ModelPerformanceSummary{ + StorageMode: "legacy_log", + Source: "log_db", + Partial: true, + QualityFlags: qualityFlags, + TimeRange: ModelPerformanceTimeRange{ + StartAt: query.StartAt, + EndAt: query.EndAt, + Hours: int((query.EndAt - query.StartAt) / int64(time.Hour/time.Second)), + }, + Summary: summary, + Items: items, + Throughput: ModelPerformanceThroughputInfo{ + CollectionState: throughput.State, + Coverage: throughput.Coverage, + }, + Truncated: truncated, + GeneratedAt: generatedAt, + } +} + +var queryModelPerformanceSummaryRange = perfmetrics.QuerySummaryAllRangeDetailed + +func queryModelPerformanceThroughput(query ModelPerformanceQuery) modelPerformanceThroughputSnapshot { + groups := []string(nil) + if query.Group != "" { + groups = []string{query.Group} + } + result, err := queryModelPerformanceSummaryRange(query.StartAt, query.EndAt, groups) + if err != nil { + return modelPerformanceThroughputSnapshot{State: perfmetrics.CollectionStateQueryFailed} + } + byModel := make(map[string]float64, len(result.Models)) + for _, summary := range result.Models { + byModel[summary.ModelName] = summary.AvgTps + } + return modelPerformanceThroughputSnapshot{ + State: result.CollectionState, + Coverage: result.Coverage, + ByModel: byModel, + } +} + +var defaultModelPerformanceReader ModelPerformanceReader = legacyModelPerformanceReader{} + +func GetModelPerformance(ctx context.Context, query ModelPerformanceQuery) (ModelPerformanceSummary, error) { + return defaultModelPerformanceReader.Summary(ctx, query) +} + +func GetModelPerformanceDetail(_ context.Context, modelName string) (perfmetrics.DetailedQueryResult, error) { + modelName = strings.TrimSpace(modelName) + if modelName == "" { + return perfmetrics.DetailedQueryResult{}, fmt.Errorf("%w: model is required", ErrInvalidModelPerformanceQuery) + } + return perfmetrics.QueryDetailed(perfmetrics.QueryParams{ + Model: modelName, + Hours: modelPerformanceDetailHours, + }) +} + +func normalizeModelPerformanceQuery(query ModelPerformanceQuery) (ModelPerformanceQuery, bool, error) { + now := time.Now().Unix() + rangeWasClamped := false + if query.EndAt <= 0 { + query.EndAt = now + } + if query.StartAt <= 0 { + hours := query.Hours + if hours <= 0 { + hours = defaultChannelPerformanceHours + } + if hours > maxChannelPerformanceHours { + hours = maxChannelPerformanceHours + rangeWasClamped = true + } + query.Hours = hours + query.StartAt = query.EndAt - int64(hours)*int64(time.Hour/time.Second) + } + if query.EndAt <= query.StartAt { + return query, false, fmt.Errorf("%w: end must be greater than start", ErrInvalidModelPerformanceQuery) + } + if query.Limit <= 0 { + query.Limit = defaultChannelPerformanceLimit + } + if query.Limit > maxChannelPerformanceLimit { + query.Limit = maxChannelPerformanceLimit + } + maxWindow := int64(maxChannelPerformanceHours) * int64(time.Hour/time.Second) + if query.EndAt-query.StartAt > maxWindow { + query.StartAt = query.EndAt - maxWindow + rangeWasClamped = true + } + return query, rangeWasClamped, nil +} diff --git a/service/model_performance_test.go b/service/model_performance_test.go new file mode 100644 index 00000000..e0574434 --- /dev/null +++ b/service/model_performance_test.go @@ -0,0 +1,176 @@ +package service + +import ( + "errors" + "testing" + "time" + + "github.com/MAX-API-Next/MAX-API/model" + perfmetrics "github.com/MAX-API-Next/MAX-API/pkg/perf_metrics" + "github.com/stretchr/testify/require" +) + +func TestNormalizeModelPerformanceQueryUsesOneHourDefaultAndClampsWindow(t *testing.T) { + before := time.Now().Unix() + query, clamped, err := normalizeModelPerformanceQuery(ModelPerformanceQuery{}) + after := time.Now().Unix() + + require.NoError(t, err) + require.False(t, clamped) + require.GreaterOrEqual(t, query.EndAt, before) + require.LessOrEqual(t, query.EndAt, after) + require.EqualValues(t, time.Hour/time.Second, query.EndAt-query.StartAt) + + query, clamped, err = normalizeModelPerformanceQuery(ModelPerformanceQuery{Hours: 37}) + require.NoError(t, err) + require.False(t, clamped) + require.EqualValues(t, 37*time.Hour/time.Second, query.EndAt-query.StartAt) + + query, clamped, err = normalizeModelPerformanceQuery(ModelPerformanceQuery{Hours: 999}) + require.NoError(t, err) + require.True(t, clamped) + require.EqualValues(t, maxChannelPerformanceHours*time.Hour/time.Second, query.EndAt-query.StartAt) +} + +func TestNormalizeModelPerformanceQueryRejectsReversedRange(t *testing.T) { + _, _, err := normalizeModelPerformanceQuery(ModelPerformanceQuery{StartAt: 20, EndAt: 10}) + + require.Error(t, err) + require.True(t, errors.Is(err, ErrInvalidModelPerformanceQuery)) +} + +func TestBuildLegacyModelPerformanceSummaryPreservesGlobalSummaryWhenRowsTruncated(t *testing.T) { + result := buildLegacyModelPerformanceSummary( + ModelPerformanceQuery{StartAt: 1_700_000_000, EndAt: 1_700_003_600, Limit: 1}, + false, + model.LegacyModelPerformanceResult{ + Rows: []model.LegacyModelPerformanceRow{ + {ModelName: "alpha"}, + {ModelName: "beta"}, + }, + Summary: model.LegacyModelPerformanceSummary{ + ModelCount: 2, + ChannelCount: 3, + ObservedCount: 5, + ConsumeLogCount: 3, + ErrorLogCount: 2, + ConsumedQuota: 900, + RetryLogCount: 1, + LatencySampleCount: 4, + TotalUseTimeSeconds: 3, + LastObservedAt: 1_700_003_500, + }, + RetryMetricsAvailable: true, + RetryMetricState: model.LegacyRetryMetricStateAvailable, + }, + []model.LegacyModelPerformanceRow{{ + ModelName: "alpha", + ChannelCount: 2, + ObservedCount: 3, + ConsumeLogCount: 2, + ErrorLogCount: 1, + ConsumedQuota: 500, + RetryLogCount: 1, + LatencySampleCount: 2, + TotalUseTimeSeconds: 1, + LastObservedAt: 1_700_003_400, + }}, + true, + true, + true, + modelPerformanceThroughputSnapshot{ + State: perfmetrics.CollectionStateAvailable, + ByModel: map[string]float64{"alpha": 42.5}, + }, + 1_700_003_600, + ) + + require.True(t, result.Truncated) + require.Len(t, result.Items, 1) + require.Contains(t, result.QualityFlags, "truncated") + require.Equal(t, 2, result.Summary.ModelCount) + require.Equal(t, 3, result.Summary.ChannelCount) + require.EqualValues(t, 5, result.Summary.ObservedCount) + require.EqualValues(t, 900, result.Summary.ConsumedQuota) + require.NotNil(t, result.Summary.ObservedSuccessRate) + require.InDelta(t, 60, *result.Summary.ObservedSuccessRate, 0.001) + require.NotNil(t, result.Summary.AvgLoggedLatencyMs) + require.InDelta(t, 750, *result.Summary.AvgLoggedLatencyMs, 0.001) + require.NotNil(t, result.Items[0].RetryLogCount) + require.EqualValues(t, 500, result.Items[0].ConsumedQuota) + require.EqualValues(t, 1, *result.Items[0].RetryLogCount) + require.NotNil(t, result.Items[0].AvgTps) + require.InDelta(t, 42.5, *result.Items[0].AvgTps, 0.001) +} + +func TestBuildLegacyModelPerformanceSummaryDoesNotInventMetricsWhenUnavailable(t *testing.T) { + result := buildLegacyModelPerformanceSummary( + ModelPerformanceQuery{StartAt: 1, EndAt: 3_601, Limit: 10}, + false, + model.LegacyModelPerformanceResult{ + Summary: model.LegacyModelPerformanceSummary{ObservedCount: 1, ConsumeLogCount: 1}, + RetryMetricState: model.LegacyRetryMetricStatePending, + }, + []model.LegacyModelPerformanceRow{{ + ModelName: "alpha", ObservedCount: 1, ConsumeLogCount: 1, + }}, + false, + true, + true, + modelPerformanceThroughputSnapshot{State: perfmetrics.CollectionStateQueryFailed}, + 3_601, + ) + + require.Nil(t, result.Summary.RetryLogCount) + require.Nil(t, result.Items[0].RetryLogCount) + require.Nil(t, result.Items[0].AvgLoggedLatencyMs) + require.Nil(t, result.Items[0].AvgTps) + require.Contains(t, result.QualityFlags, "retry_backfill_pending") + require.Contains(t, result.QualityFlags, "throughput_query_failed") +} + +func TestBuildLegacyModelPerformanceSummaryDistinguishesThroughputStates(t *testing.T) { + tests := []struct { + name string + state perfmetrics.CollectionState + flag string + }{ + {name: "collection disabled", state: perfmetrics.CollectionStateDisabled, flag: "throughput_collection_disabled"}, + {name: "no samples", state: perfmetrics.CollectionStateNoSamples, flag: "throughput_no_samples"}, + {name: "query failed", state: perfmetrics.CollectionStateQueryFailed, flag: "throughput_query_failed"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := buildLegacyModelPerformanceSummary( + ModelPerformanceQuery{StartAt: 1, EndAt: 3_601, Limit: 10}, + false, + model.LegacyModelPerformanceResult{}, + nil, + false, + true, + true, + modelPerformanceThroughputSnapshot{State: test.state}, + 3_601, + ) + + require.Contains(t, result.QualityFlags, test.flag) + require.Equal(t, test.state, result.Throughput.CollectionState) + }) + } +} + +func TestQueryModelPerformanceThroughputMarksQueryFailure(t *testing.T) { + originalQuery := queryModelPerformanceSummaryRange + queryModelPerformanceSummaryRange = func(int64, int64, []string) (perfmetrics.DetailedSummaryAllResult, error) { + return perfmetrics.DetailedSummaryAllResult{}, errors.New("database unavailable") + } + t.Cleanup(func() { + queryModelPerformanceSummaryRange = originalQuery + }) + + snapshot := queryModelPerformanceThroughput(ModelPerformanceQuery{StartAt: 1, EndAt: 3_601}) + + require.Equal(t, perfmetrics.CollectionStateQueryFailed, snapshot.State) + require.Empty(t, snapshot.ByModel) +} diff --git a/web/default/src/components/layout/config/smart-ops.config.test.ts b/web/default/src/components/layout/config/smart-ops.config.test.ts new file mode 100644 index 00000000..a52b544e --- /dev/null +++ b/web/default/src/components/layout/config/smart-ops.config.test.ts @@ -0,0 +1,45 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import type { TFunction } from 'i18next' +import assert from 'node:assert/strict' +import { describe, test } from 'node:test' +import { ROLE } from '@/lib/roles' +import { buildSmartOpsNavGroups } from './smart-ops.config' + +const translate = ((key: string) => key) as TFunction + +describe('Smart Operations navigation', () => { + test('keeps channel and model performance available to administrators', () => { + const [group] = buildSmartOpsNavGroups(translate, ROLE.ADMIN) + + assert.deepEqual( + group.items.map((item) => item.title), + ['Channel performance', 'Model performance'] + ) + }) + + test('adds system information only for the super administrator', () => { + const [group] = buildSmartOpsNavGroups(translate, ROLE.SUPER_ADMIN) + + assert.deepEqual( + group.items.map((item) => item.title), + ['Channel performance', 'Model performance', 'System Info'] + ) + }) +}) diff --git a/web/default/src/components/layout/config/smart-ops.config.ts b/web/default/src/components/layout/config/smart-ops.config.ts new file mode 100644 index 00000000..fba07f1c --- /dev/null +++ b/web/default/src/components/layout/config/smart-ops.config.ts @@ -0,0 +1,73 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import { type TFunction } from 'i18next' +import { Bot, Gauge, ServerCog } from 'lucide-react' +import { useAuthStore } from '@/stores/auth-store' +import { ROLE } from '@/lib/roles' +import type { NavGroup, SidebarView } from '../types' + +export function buildSmartOpsNavGroups( + t: TFunction, + userRole: number | undefined +): NavGroup[] { + const isSuperAdmin = userRole === ROLE.SUPER_ADMIN + const items = [ + { + title: t('Channel performance'), + url: '/smart-ops/channel-performance' as const, + icon: Gauge, + }, + { + title: t('Model performance'), + url: '/smart-ops/model-performance' as const, + icon: Bot, + }, + ...(isSuperAdmin + ? [ + { + title: t('System Info'), + url: '/smart-ops/system-info' as const, + icon: ServerCog, + }, + ] + : []), + ] + + return [ + { + id: 'smart-operations', + title: t('Smart Operations Center'), + items, + }, + ] +} + +function getSmartOpsNavGroups(t: TFunction): NavGroup[] { + return buildSmartOpsNavGroups(t, useAuthStore.getState().auth.user?.role) +} + +export const SMART_OPS_VIEW: SidebarView = { + id: 'smart-ops', + pathPattern: /^\/smart-ops(\/|$)/, + parent: { + to: '/dashboard/overview', + label: 'Back to Dashboard', + }, + getNavGroups: getSmartOpsNavGroups, +} diff --git a/web/default/src/components/layout/lib/sidebar-view-registry.ts b/web/default/src/components/layout/lib/sidebar-view-registry.ts index fde22838..54280011 100644 --- a/web/default/src/components/layout/lib/sidebar-view-registry.ts +++ b/web/default/src/components/layout/lib/sidebar-view-registry.ts @@ -17,6 +17,7 @@ along with this program. If not, see . For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues */ import { type TFunction } from 'i18next' +import { SMART_OPS_VIEW } from '../config/smart-ops.config' import { SYSTEM_SETTINGS_VIEW } from '../config/system-settings.config' import type { NavGroup, SidebarView } from '../types' @@ -29,7 +30,10 @@ import type { NavGroup, SidebarView } from '../types' * * Match priority is array order; the first matching `pathPattern` wins. */ -const SIDEBAR_VIEWS: readonly SidebarView[] = [SYSTEM_SETTINGS_VIEW] +const SIDEBAR_VIEWS: readonly SidebarView[] = [ + SMART_OPS_VIEW, + SYSTEM_SETTINGS_VIEW, +] /** * Resolve the active nested view for the given path. diff --git a/web/default/src/components/profile-dropdown.tsx b/web/default/src/components/profile-dropdown.tsx index 5b405f8a..0749121a 100644 --- a/web/default/src/components/profile-dropdown.tsx +++ b/web/default/src/components/profile-dropdown.tsx @@ -47,7 +47,9 @@ export function ProfileDropdown() { const { displayName, roleLabel } = useUserDisplay(user) const isSuperAdmin = user?.role === ROLE.SUPER_ADMIN const isWalletVisible = useIsSidebarModuleVisible('/wallet') - const isSystemInfoVisible = useIsSidebarModuleVisible('/system-info') + const isSystemInfoVisible = useIsSidebarModuleVisible( + '/smart-ops/system-info' + ) const avatarName = user?.username || displayName const avatarFallback = getUserAvatarFallback(avatarName) const avatarFallbackStyle = useMemo( @@ -116,7 +118,7 @@ export function ProfileDropdown() { {isSuperAdmin && isSystemInfoVisible && ( navigate({ to: '/system-info' })} + onClick={() => navigate({ to: '/smart-ops/system-info' })} > {t('System Info')} diff --git a/web/default/src/features/performance-metrics/types.ts b/web/default/src/features/performance-metrics/types.ts index 64469917..9714e7fe 100644 --- a/web/default/src/features/performance-metrics/types.ts +++ b/web/default/src/features/performance-metrics/types.ts @@ -33,12 +33,38 @@ export type PerformanceGroup = { series: PerformanceSeriesPoint[] } +export type PerformanceAggregate = { + avg_ttft_ms: number + avg_latency_ms: number + success_rate: number + avg_tps: number + series: PerformanceSeriesPoint[] +} + +export type PerformanceCollectionState = + | 'available' + | 'partial' + | 'collection_disabled' + | 'no_samples' + | 'query_failed' + +export type PerformanceCoverage = { + requested_start_at: number + requested_end_at: number + bucket_start_at: number + bucket_end_at: number + bucket_seconds: number + granularity_state?: 'known' | 'unknown' | 'mixed' + approximate: boolean +} + export type PerformanceMetricsData = { success: boolean message?: string data: { model_name: string series_schema?: string + summary?: PerformanceAggregate groups: PerformanceGroup[] } } diff --git a/web/default/src/features/pricing/components/model-details-chart-utils.ts b/web/default/src/features/pricing/components/model-details-chart-utils.ts new file mode 100644 index 00000000..01db9307 --- /dev/null +++ b/web/default/src/features/pricing/components/model-details-chart-utils.ts @@ -0,0 +1,92 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import type { LatencyTimePoint, UptimeDayPoint } from '../lib/mock-stats' + +export function buildLatencyChartData(series: LatencyTimePoint[]) { + return series.map((point) => ({ + time: point.timestamp, + group: point.group, + ttft: point.ttft_ms, + })) +} + +export function buildUptimeChartData(series: UptimeDayPoint[]) { + return series.map((point) => ({ + date: point.date, + uptime: point.uptime_pct, + incidents: point.incidents, + outage: point.outage_minutes, + })) +} + +export function downsampleUptimeSeries( + series: UptimeDayPoint[], + maxPoints: number +): UptimeDayPoint[] { + if (maxPoints <= 0 || series.length <= maxPoints) return series + + return Array.from({ length: maxPoints }, (_, index) => { + const start = Math.floor((index * series.length) / maxPoints) + const end = Math.max( + start + 1, + Math.floor(((index + 1) * series.length) / maxPoints) + ) + let worst = series[start] + for (let cursor = start + 1; cursor < end; cursor += 1) { + if (series[cursor].uptime_pct < worst.uptime_pct) { + worst = series[cursor] + } + } + return worst + }) +} + +export function formatChartAxisTime(iso: string): string { + const date = new Date(iso) + if (Number.isNaN(date.getTime())) return iso + return date.toLocaleTimeString(undefined, { + hour: '2-digit', + minute: '2-digit', + }) +} + +export function formatChartTooltipTime(iso: string): string { + const date = new Date(iso) + if (Number.isNaN(date.getTime())) return iso + return date.toLocaleString(undefined, { + month: 'short', + day: 'numeric', + hour: '2-digit', + minute: '2-digit', + }) +} + +export function getUptimeAxisDomain(series: UptimeDayPoint[]) { + const values = series + .map((point) => point.uptime_pct) + .filter((value) => Number.isFinite(value)) + if (values.length === 0) return { min: 95, max: 100 } + + const minValue = Math.min(...values) + const maxValue = Math.max(...values) + return { + min: Math.min(95, Math.floor(minValue / 5) * 5), + max: Math.max(100, Math.ceil(maxValue / 5) * 5), + } +} diff --git a/web/default/src/features/pricing/components/model-details-charts.tsx b/web/default/src/features/pricing/components/model-details-charts.tsx index 46f23d91..90884a35 100644 --- a/web/default/src/features/pricing/components/model-details-charts.tsx +++ b/web/default/src/features/pricing/components/model-details-charts.tsx @@ -25,27 +25,13 @@ import { cn } from '@/lib/utils' import { VCHART_OPTION } from '@/lib/vchart' import { useThemeCustomization } from '@/context/theme-customization-provider' import type { LatencyTimePoint, UptimeDayPoint } from '../lib/mock-stats' - -function formatHourLabel(iso: string): string { - const date = new Date(iso) - const hours = date.getHours() - return `${String(hours).padStart(2, '0')}:00` -} - -function formatDayLabel(date: string): string { - const parsed = new Date(date) - if (date.includes('T')) { - return parsed.toLocaleString(undefined, { - month: 'short', - day: 'numeric', - hour: '2-digit', - }) - } - return parsed.toLocaleDateString(undefined, { - month: 'short', - day: 'numeric', - }) -} +import { + buildLatencyChartData, + buildUptimeChartData, + formatChartAxisTime, + formatChartTooltipTime, + getUptimeAxisDomain, +} from './model-details-chart-utils' function getChartThemeTokens(resolvedTheme: string) { return { @@ -74,11 +60,8 @@ export function LatencyTrendChart(props: { const spec = useMemo(() => { if (props.series.length === 0) return null - const data = props.series.map((point) => ({ - time: formatHourLabel(point.timestamp), - group: point.group, - ttft: point.ttft_ms, - })) + + const data = buildLatencyChartData(props.series) return { type: 'line' as const, data: [{ id: 'latency', values: data }], @@ -96,7 +79,9 @@ export function LatencyTrendChart(props: { legends: { visible: false }, tooltip: { mark: { - title: { value: (d: { time: string }) => d.time }, + title: { + value: (d: { time: string }) => formatChartTooltipTime(d.time), + }, content: [ { key: t('Average TTFT'), @@ -109,6 +94,8 @@ export function LatencyTrendChart(props: { { orient: 'bottom', label: { + formatMethod: (val: number | string) => + formatChartAxisTime(String(val)), style: { fill: textColor, fontSize: 10 }, }, tick: { visible: false }, @@ -173,12 +160,8 @@ export function UptimeTrendChart(props: { const spec = useMemo(() => { if (props.series.length === 0) return null - const data = props.series.map((point) => ({ - date: formatDayLabel(point.date), - uptime: point.uptime_pct, - incidents: point.incidents, - outage: point.outage_minutes, - })) + const axisDomain = getUptimeAxisDomain(props.series) + const data = buildUptimeChartData(props.series) return { type: 'line' as const, @@ -204,7 +187,9 @@ export function UptimeTrendChart(props: { }, tooltip: { mark: { - title: { value: (d: { date: string }) => d.date }, + title: { + value: (d: { date: string }) => formatChartTooltipTime(d.date), + }, content: [ { key: t('Uptime'), @@ -225,6 +210,8 @@ export function UptimeTrendChart(props: { { orient: 'bottom', label: { + formatMethod: (val: number | string) => + formatChartAxisTime(String(val)), style: { fill: textColor, fontSize: 10 }, autoLimit: true, }, @@ -232,8 +219,8 @@ export function UptimeTrendChart(props: { }, { orient: 'left', - min: 95, - max: 100, + min: axisDomain.min, + max: axisDomain.max, label: { formatMethod: (val: number | string) => `${val}%`, style: { fill: textColor, fontSize: 10 }, diff --git a/web/default/src/features/pricing/components/model-details-performance.tsx b/web/default/src/features/pricing/components/model-details-performance.tsx index 7f2d2f8e..993444ee 100644 --- a/web/default/src/features/pricing/components/model-details-performance.tsx +++ b/web/default/src/features/pricing/components/model-details-performance.tsx @@ -36,7 +36,10 @@ import { formatThroughput, formatUptimePct, } from '@/features/performance-metrics/lib/format' -import type { PerformanceGroup } from '@/features/performance-metrics/types' +import type { + PerformanceAggregate, + PerformanceGroup, +} from '@/features/performance-metrics/types' import { type UptimeDayPoint } from '../lib/mock-stats' import type { PricingModel } from '../types' import { LatencyTrendChart, UptimeTrendChart } from './model-details-charts' @@ -83,55 +86,27 @@ type PerformanceRow = { avg_tps: number } -function toLatencySeries(groups: PerformanceGroup[]) { - const byTs = new Map() - for (const group of groups) { - for (const point of group.series) { - if (point.avg_ttft_ms <= 0) continue - const current = byTs.get(point.ts) ?? [] - current.push(point.avg_ttft_ms) - byTs.set(point.ts, current) - } - } - - return Array.from(byTs.entries()) - .sort(([a], [b]) => a - b) - .map(([ts, values]) => ({ - timestamp: new Date(ts * 1000).toISOString(), - group: 'latency', - ttft_ms: Math.round( - values.reduce((sum, value) => sum + value, 0) / values.length - ), +function toLatencySeriesFromAggregate(summary?: PerformanceAggregate) { + if (!summary) return [] + return summary.series + .filter((point) => point.avg_ttft_ms > 0) + .map((point) => ({ + timestamp: new Date(point.ts * 1000).toISOString(), + group: 'model', + ttft_ms: point.avg_ttft_ms, })) } -function toUptimeSeries(groups: PerformanceGroup[]): UptimeDayPoint[] { - const byTs = new Map() - for (const group of groups) { - for (const point of group.series) { - const current = byTs.get(point.ts) ?? { rates: [], incidents: 0 } - if (Number.isFinite(point.success_rate)) { - current.rates.push(point.success_rate) - if (point.success_rate < 100) current.incidents += 1 - } - byTs.set(point.ts, current) - } - } - return Array.from(byTs.entries()) - .sort(([a], [b]) => a - b) - .map(([ts, value]) => { - const uptime = - value.rates.length > 0 - ? value.rates.reduce((sum, rate) => sum + rate, 0) / - value.rates.length - : 0 - return { - date: new Date(ts * 1000).toISOString(), - uptime_pct: Math.round(uptime * 100) / 100, - incidents: value.incidents, - outage_minutes: 0, - } - }) +function toUptimeSeriesFromAggregate( + summary?: PerformanceAggregate +): UptimeDayPoint[] { + if (!summary) return [] + return summary.series.map((point) => ({ + date: new Date(point.ts * 1000).toISOString(), + uptime_pct: Math.round(point.success_rate * 100) / 100, + incidents: point.success_rate < 100 ? 1 : 0, + outage_minutes: 0, + })) } function toGroupUptimeSeries(group: PerformanceGroup): UptimeDayPoint[] { @@ -143,19 +118,7 @@ function toGroupUptimeSeries(group: PerformanceGroup): UptimeDayPoint[] { })) } -function average( - rows: PerformanceRow[], - field: 'avg_ttft_ms' | 'avg_latency_ms' -) { - const values = rows.map((row) => row[field]).filter((value) => value > 0) - if (values.length === 0) return 0 - return Math.round( - values.reduce((sum, value) => sum + value, 0) / values.length - ) -} - export function ModelDetailsPerformance(props: { model: PricingModel }) { - const { t } = useTranslation() const metricsQuery = useQuery({ queryKey: ['perf-metrics', props.model.model_name], queryFn: () => getPerfMetrics(props.model.model_name, 24), @@ -165,6 +128,24 @@ export function ModelDetailsPerformance(props: { model: PricingModel }) { () => metricsQuery.data?.data.groups ?? [], [metricsQuery.data] ) + const summary = metricsQuery.data?.data.summary + + return ( + + ) +} + +export function ModelPerformanceDetailsContent(props: { + groups: PerformanceGroup[] + summary?: PerformanceAggregate + loading?: boolean +}) { + const { t } = useTranslation() + const groups = props.groups const performances = useMemo( () => groups.map((group) => ({ @@ -176,8 +157,26 @@ export function ModelDetailsPerformance(props: { model: PricingModel }) { })), [groups] ) - const latencySeries = useMemo(() => toLatencySeries(groups), [groups]) - const uptimeSeries = useMemo(() => toUptimeSeries(groups), [groups]) + const aggregate = useMemo(() => { + if (props.summary) return props.summary + if (groups.length !== 1) return undefined + const group = groups[0] + return { + avg_ttft_ms: group.avg_ttft_ms, + avg_latency_ms: group.avg_latency_ms, + success_rate: group.success_rate, + avg_tps: group.avg_tps, + series: group.series, + } + }, [groups, props.summary]) + const latencySeries = useMemo( + () => (aggregate ? toLatencySeriesFromAggregate(aggregate) : []), + [aggregate] + ) + const uptimeSeries = useMemo( + () => (aggregate ? toUptimeSeriesFromAggregate(aggregate) : []), + [aggregate] + ) const uptimeByGroup = useMemo>(() => { const map: Record = {} for (const group of groups) { @@ -186,7 +185,7 @@ export function ModelDetailsPerformance(props: { model: PricingModel }) { return map }, [groups]) - if (metricsQuery.isLoading || performances.length === 0) { + if (props.loading || performances.length === 0) { return (
{t('Performance data is not yet available for this model.')} @@ -194,22 +193,9 @@ export function ModelDetailsPerformance(props: { model: PricingModel }) { ) } - const tpsValues = performances - .map((p) => p.avg_tps) - .filter((value) => value > 0) - const avgTps = - tpsValues.length > 0 - ? tpsValues.reduce((sum, value) => sum + value, 0) / tpsValues.length - : 0 - const avgLatency = average(performances, 'avg_latency_ms') - const successRates = performances - .map((perf) => perf.success_rate) - .filter((value) => Number.isFinite(value)) - const successRate = - successRates.length > 0 - ? successRates.reduce((sum, value) => sum + value, 0) / - successRates.length - : 0 + const avgTps = aggregate?.avg_tps ?? Number.NaN + const avgLatency = aggregate?.avg_latency_ms ?? Number.NaN + const successRate = aggregate?.success_rate ?? Number.NaN const incidentCount = uptimeSeries.reduce((s, p) => s + p.incidents, 0) let intent: 'default' | 'warning' | 'success' = 'warning' if (successRate >= 99.9) { @@ -296,6 +282,7 @@ export function ModelDetailsPerformance(props: { model: PricingModel }) { diff --git a/web/default/src/features/pricing/components/model-details-uptime-sparkline.tsx b/web/default/src/features/pricing/components/model-details-uptime-sparkline.tsx index 6bdbe747..366767c3 100644 --- a/web/default/src/features/pricing/components/model-details-uptime-sparkline.tsx +++ b/web/default/src/features/pricing/components/model-details-uptime-sparkline.tsx @@ -27,6 +27,7 @@ import { } from '@/components/ui/tooltip' import { formatUptimePct } from '@/features/performance-metrics/lib/format' import { aggregateUptime, type UptimeDayPoint } from '../lib/mock-stats' +import { downsampleUptimeSeries } from './model-details-chart-utils' // --------------------------------------------------------------------------- // Uptime sparkline @@ -46,6 +47,7 @@ type UptimeSparklineProps = { series: UptimeDayPoint[] size?: SparklineSize showOverall?: boolean + overallPct?: number emptyLabel?: string className?: string } @@ -83,8 +85,15 @@ export function UptimeSparkline(props: UptimeSparklineProps) { ) } - const overall = + const calculatedOverall = props.series.reduce((s, p) => s + p.uptime_pct, 0) / props.series.length + const overall = Number.isFinite(props.overallPct) + ? (props.overallPct as number) + : calculatedOverall + const displaySeries = downsampleUptimeSeries( + props.series, + size === 'sm' ? 32 : 48 + ) const containerHeight = size === 'sm' ? 'h-3.5' : 'h-5' const barWidth = size === 'sm' ? 'w-[3px]' : 'w-1' @@ -99,7 +108,7 @@ export function UptimeSparkline(props: UptimeSparklineProps) { percent: overall.toFixed(2), })} > - {props.series.map((day) => ( + {displaySeries.map((day) => ( {t('Last 30 days uptime')}
- +
{statusLabel} diff --git a/web/default/src/features/smart-ops/api.ts b/web/default/src/features/smart-ops/api.ts new file mode 100644 index 00000000..435f53bb --- /dev/null +++ b/web/default/src/features/smart-ops/api.ts @@ -0,0 +1,88 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import { api } from '@/lib/api' +import type { + ChannelPerformanceData, + ChannelPerformanceQuery, + ChannelPerformanceResponse, + ModelPerformanceData, + ModelPerformanceDetailData, + ModelPerformanceDetailResponse, + ModelPerformanceQuery, + ModelPerformanceResponse, +} from './types' + +export async function getChannelPerformance( + query: ChannelPerformanceQuery +): Promise { + const response = await api.get( + '/api/smart-ops/channel-performance', + { + params: { + hours: query.hours, + limit: 200, + channel_id: query.channelId, + model: query.model || undefined, + group: query.group || undefined, + }, + } + ) + return response.data.data +} + +export async function getChannelPerformanceDetail( + channelId: number +): Promise { + const response = await api.get( + '/api/smart-ops/channel-performance/detail', + { + params: { channel_id: channelId }, + } + ) + return response.data.data +} + +export async function getModelPerformanceDetail( + modelName: string +): Promise { + const response = await api.get( + '/api/smart-ops/model-performance/detail', + { + params: { model: modelName }, + } + ) + return response.data.data +} + +export async function getModelPerformance( + query: ModelPerformanceQuery +): Promise { + const response = await api.get( + '/api/smart-ops/model-performance', + { + params: { + hours: query.hours, + limit: 200, + model: query.model || undefined, + group: query.group || undefined, + }, + } + ) + return response.data.data +} diff --git a/web/default/src/features/smart-ops/index.test.tsx b/web/default/src/features/smart-ops/index.test.tsx new file mode 100644 index 00000000..a95dbcc1 --- /dev/null +++ b/web/default/src/features/smart-ops/index.test.tsx @@ -0,0 +1,576 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import { act } from 'react' +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { createReactTestEnvironment } from '@/test/react' +import { waitFor } from '@testing-library/react' +import assert from 'node:assert/strict' +import { after, before, describe, test } from 'node:test' +import { api } from '@/lib/api' +import { formatQuota } from '@/lib/format' +import { ProductionPerformance } from './index' +import { getChannelDetailObservation } from './lib/channel-detail' +import type { ChannelPerformanceData } from './types' + +const testEnv = createReactTestEnvironment() + +before(() => testEnv.setup()) + +after(() => testEnv.teardown()) + +const emptyPerformanceData: ChannelPerformanceData = { + storage_mode: 'legacy_log', + source: 'log_db', + partial: true, + quality_flags: ['legacy'], + time_range: { start_at: 1, end_at: 3601, hours: 1 }, + summary: { + channel_count: 0, + observed_count: 0, + consume_log_count: 0, + error_log_count: 0, + consumed_quota: 0, + retry_log_count: 0, + latency_sample_count: 0, + observed_success_rate: null, + avg_logged_latency_ms: null, + last_observed_at: 0, + }, + items: [], + truncated: false, + generated_at: 3601, +} + +const performanceDataWithErrors: ChannelPerformanceData = { + ...emptyPerformanceData, + summary: { + ...emptyPerformanceData.summary, + channel_count: 1, + observed_count: 10, + consume_log_count: 7, + error_log_count: 3, + consumed_quota: 750000, + observed_success_rate: 70, + last_observed_at: 3600, + }, + items: [ + { + channel_id: 12, + channel_name: 'Test channel', + channel_type: 1, + channel_status: 1, + model_name: 'test-model', + effective_group: 'default', + observed_count: 10, + consume_log_count: 7, + error_log_count: 3, + consumed_quota: 750000, + retry_log_count: 0, + latency_sample_count: 7, + observed_success_rate: 70, + avg_logged_latency_ms: 1000, + avg_tps: null, + last_observed_at: 3600, + probe_latency_ms: 100, + probe_test_time: 3500, + quality_flags: ['legacy'], + }, + ], +} + +const performanceDataWithDisabledErrorLogs: ChannelPerformanceData = { + ...performanceDataWithErrors, + quality_flags: [ + 'legacy', + 'partial', + 'weak_correlation', + 'heuristic_outcome', + 'coarse_latency', + 'probe_separate', + 'non_attempt_logs_excluded', + 'incomplete_errors_possible', + 'error_logs_disabled', + ], + summary: { + ...performanceDataWithErrors.summary, + error_log_count: 0, + observed_success_rate: null, + }, + items: performanceDataWithErrors.items.map((item) => ({ + ...item, + error_log_count: 0, + observed_success_rate: null, + quality_flags: [ + ...item.quality_flags, + 'incomplete_errors_possible', + 'error_logs_disabled', + ], + })), +} + +const performanceDataWithSortableRows: ChannelPerformanceData = { + ...performanceDataWithErrors, + items: [ + { + ...performanceDataWithErrors.items[0], + channel_id: 21, + channel_name: 'Healthy channel', + observed_count: 20, + consume_log_count: 20, + error_log_count: 0, + consumed_quota: 1000000, + observed_success_rate: 100, + avg_logged_latency_ms: 300, + retry_log_count: 0, + probe_latency_ms: 80, + }, + { + ...performanceDataWithErrors.items[0], + channel_id: 22, + channel_name: 'Problem channel', + observed_count: 12, + consume_log_count: 4, + error_log_count: 8, + consumed_quota: 200000, + observed_success_rate: 33.333, + avg_logged_latency_ms: 5000, + retry_log_count: 6, + probe_latency_ms: 1200, + }, + { + ...performanceDataWithErrors.items[0], + channel_id: 23, + channel_name: 'Unknown channel', + observed_count: 2, + consume_log_count: 2, + error_log_count: 0, + consumed_quota: 0, + observed_success_rate: null, + avg_logged_latency_ms: null, + retry_log_count: null, + probe_latency_ms: null, + }, + ], +} + +const channelDetailData: ChannelPerformanceData = { + ...performanceDataWithErrors, + quality_flags: ['error_logs_disabled'], + time_range: { start_at: 1, end_at: 86401, hours: 24 }, + summary: { + ...performanceDataWithErrors.summary, + observed_count: 15, + consume_log_count: 12, + error_log_count: 3, + consumed_quota: 900000, + observed_success_rate: 80, + last_observed_at: 7200, + }, + items: [ + performanceDataWithErrors.items[0], + { + ...performanceDataWithErrors.items[0], + model_name: 'second-model', + effective_group: 'premium', + observed_count: 5, + consume_log_count: 5, + error_log_count: 0, + consumed_quota: 150000, + observed_success_rate: 100, + }, + ], +} + +describe('ProductionPerformance manual queries', () => { + test('requests data only after explicit user actions', async () => { + const originalGet = api.get + let requestCount = 0 + api.get = (async () => { + requestCount += 1 + return { data: { success: true, data: emptyPerformanceData } } + }) as typeof api.get + + const queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false }, + mutations: { retry: false }, + }, + }) + const view = await testEnv.render( + + + + ) + + try { + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 0)) + }) + assert.equal(requestCount, 0, 'mounting the page must not query logs') + + const findButton = (label: string) => { + const button = Array.from( + view.container.querySelectorAll('button') + ).find((candidate) => candidate.textContent?.trim() === label) + assert.ok(button, `expected ${label} button`) + return button + } + + await view.click(findButton('Apply filters')) + await waitFor(() => assert.equal(requestCount, 1)) + + await view.click(findButton('Apply filters')) + await waitFor(() => assert.equal(requestCount, 2)) + + await act(async () => { + window.dispatchEvent(new Event('focus')) + window.dispatchEvent(new Event('online')) + await new Promise((resolve) => setTimeout(resolve, 0)) + }) + assert.equal(requestCount, 2, 'focus and reconnect must not query logs') + + await view.click(findButton('Refresh')) + await waitFor(() => assert.equal(requestCount, 3)) + } finally { + api.get = originalGet + queryClient.clear() + await view.unmount() + } + }) + + test('shows error counts and consumed quota in the channel performance table', async () => { + const originalGet = api.get + api.get = (async () => ({ + data: { success: true, data: performanceDataWithErrors }, + })) as typeof api.get + + const queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false }, + mutations: { retry: false }, + }, + }) + const view = await testEnv.render( + + + + ) + + try { + const applyButton = Array.from( + view.container.querySelectorAll('button') + ).find((candidate) => candidate.textContent?.trim() === 'Apply filters') + assert.ok(applyButton, 'expected Apply filters button') + + await view.click(applyButton) + await waitFor(() => { + assert.ok( + Array.from( + view.container.querySelectorAll('[data-slot="card-description"]') + ).some((node) => node.textContent?.trim() === 'Recorded error logs') + ) + }) + + const errorCardLabel = Array.from( + view.container.querySelectorAll('[data-slot="card-description"]') + ).find((node) => node.textContent?.trim() === 'Recorded error logs') + assert.ok(errorCardLabel, 'expected error summary card') + assert.equal( + errorCardLabel.parentElement?.querySelector('[data-slot="card-title"]') + ?.textContent, + '3' + ) + + const headers = Array.from(view.container.querySelectorAll('thead th')) + const errorColumnIndex = headers.findIndex( + (header) => header.textContent?.trim() === 'Recorded error logs' + ) + assert.ok(errorColumnIndex >= 0, 'expected error count table column') + const rowCells = view.container.querySelectorAll( + 'tbody tr:first-child td' + ) + assert.equal(rowCells.item(errorColumnIndex).textContent?.trim(), '3') + const consumedQuotaColumnIndex = headers.findIndex( + (header) => header.textContent?.trim() === 'Consumed quota' + ) + assert.ok(consumedQuotaColumnIndex >= 0, 'expected consumed quota column') + assert.equal( + rowCells.item(consumedQuotaColumnIndex).textContent?.trim(), + formatQuota(750000) + ) + } finally { + api.get = originalGet + queryClient.clear() + await view.unmount() + } + }) + + test('keeps recorded errors visible and warns when error logging is disabled', async () => { + const originalGet = api.get + api.get = (async () => ({ + data: { + success: true, + data: performanceDataWithDisabledErrorLogs, + }, + })) as typeof api.get + + const queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false }, + mutations: { retry: false }, + }, + }) + const view = await testEnv.render( + + + + ) + + try { + const applyButton = Array.from( + view.container.querySelectorAll('button') + ).find((candidate) => candidate.textContent?.trim() === 'Apply filters') + assert.ok(applyButton, 'expected Apply filters button') + + await view.click(applyButton) + await waitFor(() => { + assert.ok( + Array.from( + view.container.querySelectorAll('[data-slot="alert-title"]') + ).some( + (node) => + node.textContent?.trim() === 'Error log collection is disabled' + ) + ) + }) + + const errorCardLabel = Array.from( + view.container.querySelectorAll('[data-slot="card-description"]') + ).find((node) => node.textContent?.trim() === 'Recorded error logs') + assert.ok(errorCardLabel, 'expected recorded error summary card') + assert.equal( + errorCardLabel.parentElement?.querySelector('[data-slot="card-title"]') + ?.textContent, + '0' + ) + + const successCardLabel = Array.from( + view.container.querySelectorAll('[data-slot="card-description"]') + ).find((node) => node.textContent?.trim() === 'Estimated success rate') + assert.ok(successCardLabel, 'expected success-rate summary card') + assert.equal( + successCardLabel.parentElement?.querySelector( + '[data-slot="card-title"]' + )?.textContent, + 'N/A' + ) + + assert.ok( + Array.from(view.container.querySelectorAll('[data-slot="badge"]')).some( + (node) => node.textContent?.trim() === 'Error logs disabled' + ), + 'critical log collection state must not be hidden by badge truncation' + ) + } finally { + api.get = originalGet + queryClient.clear() + await view.unmount() + } + }) + + test('provides a semantic details button without making the table row tabbable', async () => { + const originalGet = api.get + api.get = (async () => ({ + data: { success: true, data: performanceDataWithErrors }, + })) as typeof api.get + + const queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false }, + mutations: { retry: false }, + }, + }) + const view = await testEnv.render( + + + + ) + + try { + const applyButton = Array.from( + view.container.querySelectorAll('button') + ).find((candidate) => candidate.textContent?.trim() === 'Apply filters') + assert.ok(applyButton, 'expected Apply filters button') + await view.click(applyButton) + + const detailsButton = await waitFor(() => { + const button = Array.from( + view.container.querySelectorAll('tbody button') + ).find((candidate) => candidate.textContent?.trim() === 'Details') + assert.ok(button, 'expected semantic details button') + return button + }) + assert.equal( + detailsButton.getAttribute('aria-label'), + 'View details: Test channel · test-model' + ) + assert.equal( + detailsButton.closest('tr')?.getAttribute('tabindex'), + null, + 'the row must not compete with the semantic button in keyboard order' + ) + assert.equal(detailsButton.tagName, 'BUTTON') + assert.equal(detailsButton.getAttribute('type'), 'button') + } finally { + api.get = originalGet + queryClient.clear() + await view.unmount() + } + }) + + test('loads a channel-scoped 24-hour detail only when the detail sheet opens', async () => { + const originalGet = api.get + const urls: string[] = [] + api.get = (async (url, config) => { + urls.push(String(url)) + if (url === '/api/smart-ops/channel-performance/detail') { + assert.deepEqual(config?.params, { channel_id: 12 }) + return { data: { success: true, data: channelDetailData } } + } + return { data: { success: true, data: performanceDataWithErrors } } + }) as typeof api.get + + const queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false }, + mutations: { retry: false }, + }, + }) + const view = await testEnv.render( + + + + ) + + try { + assert.deepEqual(urls, []) + const applyButton = Array.from( + view.container.querySelectorAll('button') + ).find((candidate) => candidate.textContent?.trim() === 'Apply filters') + assert.ok(applyButton) + await view.click(applyButton) + await waitFor(() => assert.equal(urls.length, 1)) + assert.equal(urls[0], '/api/smart-ops/channel-performance') + + const detailsButton = await waitFor(() => { + const button = Array.from( + view.container.querySelectorAll('tbody button') + ).find((candidate) => candidate.textContent?.trim() === 'Details') + assert.ok(button) + return button + }) + await view.click(detailsButton) + + await waitFor(() => { + assert.equal(urls.length, 2) + assert.equal(urls[1], '/api/smart-ops/channel-performance/detail') + }) + } finally { + api.get = originalGet + queryClient.clear() + await view.unmount() + } + }) + + test('uses channel-scoped detail evidence instead of the clicked list row', () => { + const observation = getChannelDetailObservation( + performanceDataWithErrors.items[0], + channelDetailData + ) + + assert.equal( + observation.lastObservedAt, + channelDetailData.summary.last_observed_at + ) + assert.deepEqual(observation.qualityFlags, channelDetailData.quality_flags) + }) + + test('exposes an accessible sort menu for every performance metric', async () => { + const originalGet = api.get + api.get = (async () => ({ + data: { success: true, data: performanceDataWithSortableRows }, + })) as typeof api.get + + const queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false }, + mutations: { retry: false }, + }, + }) + const view = await testEnv.render( + + + + ) + + try { + const applyButton = Array.from( + view.container.querySelectorAll('button') + ).find((candidate) => candidate.textContent?.trim() === 'Apply filters') + assert.ok(applyButton, 'expected Apply filters button') + await view.click(applyButton) + + const sortableLabels = [ + 'Observed log events', + 'Recorded error logs', + 'Consumed quota', + 'Estimated success rate', + 'Logged latency', + 'Retries', + 'Probe latency', + ] + const headerButtons = await waitFor(() => { + const buttons = Array.from( + view.container.querySelectorAll('thead button') + ) + for (const label of sortableLabels) { + assert.ok( + buttons.some((button) => button.textContent?.trim() === label), + `expected sortable ${label} header` + ) + } + return buttons + }) + + const successRateHeader = headerButtons.find( + (button) => button.textContent?.trim() === 'Estimated success rate' + ) + assert.ok(successRateHeader, 'expected success-rate sort trigger') + for (const button of headerButtons) { + if (!sortableLabels.includes(button.textContent?.trim() ?? '')) continue + assert.equal(button.getAttribute('aria-haspopup'), 'menu') + assert.equal(button.closest('th')?.getAttribute('aria-sort'), 'none') + } + } finally { + api.get = originalGet + queryClient.clear() + await view.unmount() + } + }) +}) diff --git a/web/default/src/features/smart-ops/index.tsx b/web/default/src/features/smart-ops/index.tsx new file mode 100644 index 00000000..cb5f91b3 --- /dev/null +++ b/web/default/src/features/smart-ops/index.tsx @@ -0,0 +1,1000 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import { useMemo, useState } from 'react' +import { useMutation } from '@tanstack/react-query' +import { + AlertTriangle, + ArrowDown, + ArrowUp, + Clock3, + ChevronsUpDown, + DatabaseZap, + HeartPulse, + RefreshCw, + Search, + ShieldCheck, +} from 'lucide-react' +import { useTranslation } from 'react-i18next' +import { formatQuota } from '@/lib/format' +import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert' +import { Badge } from '@/components/ui/badge' +import { Button } from '@/components/ui/button' +import { + Card, + CardAction, + CardDescription, + CardHeader, + CardTitle, +} from '@/components/ui/card' +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuGroup, + DropdownMenuItem, + DropdownMenuTrigger, +} from '@/components/ui/dropdown-menu' +import { + Empty, + EmptyDescription, + EmptyHeader, + EmptyMedia, + EmptyTitle, +} from '@/components/ui/empty' +import { Field, FieldGroup, FieldLabel } from '@/components/ui/field' +import { Input } from '@/components/ui/input' +import { + Sheet, + SheetContent, + SheetDescription, + SheetHeader, + SheetTitle, +} from '@/components/ui/sheet' +import { Skeleton } from '@/components/ui/skeleton' +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@/components/ui/table' +import { ErrorState } from '@/components/error-state' +import { SectionPageLayout } from '@/components/layout' +import { + formatLatency, + formatUptimePct, +} from '@/features/performance-metrics/lib/format' +import { getChannelPerformance, getChannelPerformanceDetail } from './api' +import { getChannelDetailObservation } from './lib/channel-detail' +import { + DEFAULT_FILTERS, + MAX_PERFORMANCE_HOURS, + normalizeFilters, + toQuery, +} from './lib/filters' +import { formatLegacyLatency } from './lib/format' +import { + sortChannelPerformanceItems, + type ChannelPerformanceSortKey, + type ChannelPerformanceSortState, +} from './lib/sort' +import type { ChannelPerformanceData, ChannelPerformanceItem } from './types' + +const priorityQualityFlags = [ + 'error_logs_disabled', + 'consume_logs_disabled', + 'retry_backfill_pending', + 'retry_metrics_unavailable', + 'time_range_clamped', + 'truncated', + 'incomplete_errors_possible', +] + +function getVisibleQualityFlags(flags: string[]) { + const flagSet = new Set(flags) + const priorityFlags = priorityQualityFlags.filter((flag) => flagSet.has(flag)) + const otherFlags = flags.filter( + (flag) => !priorityQualityFlags.includes(flag) + ) + return [ + ...priorityFlags, + ...otherFlags.slice(0, Math.max(0, 4 - priorityFlags.length)), + ] +} + +function statusBadge( + status: number | null, + enabledLabel: string, + disabledLabel: string +) { + if (status == null) return — + return status === 1 ? ( + {enabledLabel} + ) : ( + {disabledLabel} + ) +} + +export function ProductionPerformance() { + const { t, i18n } = useTranslation() + const [draft, setDraft] = useState(DEFAULT_FILTERS) + const [filters, setFilters] = useState(DEFAULT_FILTERS) + const [selected, setSelected] = useState(null) + const [sortState, setSortState] = useState(null) + const queryParams = useMemo(() => toQuery(filters), [filters]) + const numberFormatter = useMemo( + () => new Intl.NumberFormat(i18n.language), + [i18n.language] + ) + + const performanceQuery = useMutation({ + mutationKey: ['smart-ops', 'channel-performance'], + mutationFn: getChannelPerformance, + retry: false, + }) + const detailQuery = useMutation({ + mutationKey: ['smart-ops', 'channel-performance-detail'], + mutationFn: getChannelPerformanceDetail, + retry: false, + }) + + const data = performanceQuery.data + const selectedDetailObservation = selected + ? getChannelDetailObservation(selected, detailQuery.data) + : null + const sortedItems = useMemo( + () => sortChannelPerformanceItems(data?.items ?? [], sortState), + [data?.items, sortState] + ) + const latestObservedAt = data?.summary.last_observed_at ?? 0 + const retryBackfillPending = + data?.quality_flags.includes('retry_backfill_pending') ?? false + const retryMetricsUnavailable = + data?.quality_flags.includes('retry_metrics_unavailable') ?? false + const errorLogsDisabled = + data?.quality_flags.includes('error_logs_disabled') ?? false + + const openDetails = (item: ChannelPerformanceItem) => { + detailQuery.reset() + setSelected(item) + detailQuery.mutate(item.channel_id) + } + + const formatTimestamp = (timestamp: number | null | undefined) => { + if (!timestamp) return '—' + return new Intl.DateTimeFormat(i18n.language, { + dateStyle: 'medium', + timeStyle: 'medium', + }).format(new Date(timestamp * 1000)) + } + + const qualityLabel = (flag: string) => { + const labels: Record = { + legacy: t('Legacy log'), + partial: t('Partial data'), + weak_correlation: t('Weak correlation'), + heuristic_outcome: t('Heuristic outcome'), + coarse_latency: t('Coarse latency'), + probe_separate: t('Probe data is separate'), + non_attempt_logs_excluded: t('Known non-attempt logs excluded'), + time_range_clamped: t('Time range was limited'), + truncated: t('Results truncated'), + retry_backfill_pending: t('Retry marker backfill pending'), + retry_metrics_unavailable: t('Retry metrics unavailable'), + consume_logs_disabled: t('Consumption logs disabled'), + error_logs_disabled: t('Error logs disabled'), + incomplete_errors_possible: t('Errors may be incomplete'), + metadata_missing: t('Channel metadata missing'), + probe_unavailable: t('Probe unavailable'), + } + return labels[flag] ?? flag + } + + return ( + + + + {t('Channel performance')} + {t('Read only')} + {t('Legacy log')} + + + + + + +
+ + + + {(retryBackfillPending || retryMetricsUnavailable) && ( + + + )} + + {errorLogsDisabled && ( + + + )} + +
{ + event.preventDefault() + const normalizedDraft = normalizeFilters(draft) + const nextQuery = toQuery(normalizedDraft) + setDraft(normalizedDraft) + setFilters(normalizedDraft) + performanceQuery.mutate(nextQuery) + }} + > + + + + {t('Time range')} ({t('hours')}, 1–{MAX_PERFORMANCE_HOURS}) + + + setDraft((current) => ({ + ...current, + hours: event.target.value, + })) + } + /> + + + + {t('Channel ID')} + + + setDraft((current) => ({ + ...current, + channelId: event.target.value, + })) + } + /> + + + + {t('Model')} + + + setDraft((current) => ({ + ...current, + model: event.target.value, + })) + } + /> + + + + {t('Group')} + + + setDraft((current) => ({ + ...current, + group: event.target.value, + })) + } + /> + + + +
+ +
+ + + + + +
+ + {performanceQuery.isPending ? ( + + ) : performanceQuery.isError ? ( + performanceQuery.mutate(queryParams)} + /> + ) : performanceQuery.isIdle ? ( + + + + + + {t('Channel performance is not loaded automatically')} + + + {t( + 'Select Apply filters or Refresh to query the current time range.' + )} + + + + ) : !data?.items.length ? ( + + + + + {t('No recorded channel logs')} + + {t( + 'Try a wider time range or check whether consumption and error logging are enabled.' + )} + + + + ) : ( +
+
+ + {t('Channel performance')} + + + {t('{{count}} rows', { count: data.items.length })} + +
+ {getVisibleQualityFlags(data.quality_flags).map((flag) => ( + + {qualityLabel(flag)} + + ))} +
+
+ + + + {t('Channel')} + {t('Model')} + {t('Group')} + + + + + + + + {t('Last observed')} + {t('Actions')} + + + + {sortedItems.map((item) => ( + openDetails(item)} + > + +
+
+
+ {item.channel_name} +
+
+ #{item.channel_id} +
+
+ {statusBadge( + item.channel_status, + t('Enabled'), + t('Disabled') + )} +
+ + + {item.model_name} + + {item.effective_group || '—'} + + {numberFormatter.format(item.observed_count)} + + + {numberFormatter.format(item.error_log_count)} + + + {formatQuota(item.consumed_quota)} + + + {item.observed_success_rate == null + ? t('N/A') + : formatUptimePct(item.observed_success_rate)} + + + {formatLegacyLatency(item.avg_logged_latency_ms)} + + + {item.retry_log_count == null + ? t('N/A') + : numberFormatter.format(item.retry_log_count)} + + + {formatLatency(item.probe_latency_ms ?? Number.NaN)} + + + {formatTimestamp(item.last_observed_at)} + + + + + + ))} + +
+ {data.truncated && ( +
+ {t( + 'The table was truncated to {{count}} rows. Summary cards still cover all matching eligible logs.', + { count: data.items.length } + )} +
+ )} +
+ )} + + {data && ( +
+ {t('Generated at {{time}}', { + time: formatTimestamp(data.generated_at), + })} +
+ )} + { + if (!open) { + setSelected(null) + detailQuery.reset() + } + }} + > + + + {t('Channel performance details')} + + {selected + ? `${selected.channel_name} · #${selected.channel_id}` + : ''} + + + {selected && ( +
+ +
+
+ {t('Data quality')} +
+
+ {selectedDetailObservation?.qualityFlags.length ? ( + selectedDetailObservation.qualityFlags.map((flag) => ( + + {qualityLabel(flag)} + + )) + ) : ( + {t('N/A')} + )} +
+
+ + + {detailQuery.isPending ? ( + + ) : detailQuery.isError ? ( + detailQuery.mutate(selected.channel_id)} + /> + ) : detailQuery.data ? ( + + ) : null} +
+ )} + + +
+ + + ) +} + +function SortablePerformanceHead({ + sortKey, + title, + sortState, + onSort, +}: { + sortKey: ChannelPerformanceSortKey + title: string + sortState: ChannelPerformanceSortState + onSort: (sortState: ChannelPerformanceSortState) => void +}) { + const { t } = useTranslation() + const direction = sortState?.key === sortKey ? sortState.direction : null + const ariaSort = + direction === 'asc' + ? 'ascending' + : direction === 'desc' + ? 'descending' + : 'none' + + return ( + +
+ + + } + > + {title} + {direction === 'desc' ? ( + + + + onSort({ key: sortKey, direction: 'asc' })} + > + + onSort({ key: sortKey, direction: 'desc' })} + > + + + + +
+ + ) +} + +function MetricCard({ + icon: Icon, + label, + value, + loading, +}: { + icon: typeof DatabaseZap + label: string + value: string + loading: boolean +}) { + return ( + + + {label} + + {loading ? : value} + + +
+
+ + + + ) +} + +function PerformanceTableSkeleton() { + return ( + + ) +} + +function ChannelPerformanceDetail({ + data, + numberFormatter, + formatTimestamp, +}: { + data: ChannelPerformanceData + numberFormatter: Intl.NumberFormat + formatTimestamp: (timestamp: number | null | undefined) => string +}) { + const { t } = useTranslation() + const summary = data.summary + + return ( +
+
+
+
{t('Last 24 hours')}
+
+ {t('Channel models and groups from eligible production logs')} +
+
+ + {t('{{count}} rows', { count: data.items.length })} + +
+ + {data.items.length === 0 ? ( + + + + + {t('No channel model performance')} + + {t( + 'No eligible production logs were recorded for this channel in the last 24 hours.' + )} + + + + ) : ( +
+ + + + {t('Model')} + {t('Group')} + + {t('Observed log events')} + + + {t('Recorded error logs')} + + + {t('Estimated success rate')} + + + {t('Consumed quota')} + + {t('Retries')} + + {t('Logged latency')} + + {t('Last observed')} + + + + {data.items.map((item) => ( + + + {item.model_name} + + {item.effective_group || '—'} + + {numberFormatter.format(item.observed_count)} + + + {numberFormatter.format(item.error_log_count)} + + + {item.observed_success_rate == null + ? t('N/A') + : formatUptimePct(item.observed_success_rate)} + + + {formatQuota(item.consumed_quota)} + + + {item.retry_log_count == null + ? t('N/A') + : numberFormatter.format(item.retry_log_count)} + + + {formatLegacyLatency(item.avg_logged_latency_ms)} + + + {formatTimestamp(item.last_observed_at)} + + + ))} + +
+
+ )} + {data.truncated && ( +
+ {t( + 'The detail table was truncated to {{count}} rows. The summary still covers all matching eligible logs.', + { count: data.items.length } + )} +
+ )} +
+ ) +} + +function ChannelPerformanceDetailSkeleton() { + return ( +
+ +
+ {Array.from({ length: 6 }).map((_, index) => ( + + ))} +
+ +
+ ) +} + +function DetailGrid({ rows }: { rows: [string, string][] }) { + return ( +
+ {rows.map(([label, value]) => ( +
+
{label}
+
+ {value} +
+
+ ))} +
+ ) +} diff --git a/web/default/src/features/smart-ops/lib/channel-detail.ts b/web/default/src/features/smart-ops/lib/channel-detail.ts new file mode 100644 index 00000000..830c0a4c --- /dev/null +++ b/web/default/src/features/smart-ops/lib/channel-detail.ts @@ -0,0 +1,30 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import type { ChannelPerformanceData, ChannelPerformanceItem } from '../types' + +export function getChannelDetailObservation( + selected: ChannelPerformanceItem, + detail: ChannelPerformanceData | undefined +) { + return { + lastObservedAt: detail?.summary.last_observed_at ?? null, + qualityFlags: detail?.quality_flags ?? [], + probeLatencyMs: selected.probe_latency_ms, + } +} diff --git a/web/default/src/features/smart-ops/lib/filters.test.ts b/web/default/src/features/smart-ops/lib/filters.test.ts new file mode 100644 index 00000000..40d2c29c --- /dev/null +++ b/web/default/src/features/smart-ops/lib/filters.test.ts @@ -0,0 +1,60 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import assert from 'node:assert/strict' +import { describe, test } from 'node:test' +import { + DEFAULT_FILTERS, + DEFAULT_MODEL_FILTERS, + DEFAULT_PERFORMANCE_HOURS, + MAX_PERFORMANCE_HOURS, + normalizeFilters, + normalizePerformanceHours, + toModelQuery, + toQuery, +} from './filters' + +describe('production performance filters', () => { + test('defaults the initial view and invalid fallback to the latest hour', () => { + assert.equal(DEFAULT_FILTERS.hours, '1') + assert.equal(toQuery(DEFAULT_FILTERS).hours, DEFAULT_PERFORMANCE_HOURS) + assert.equal( + toQuery({ ...DEFAULT_FILTERS, hours: '' }).hours, + DEFAULT_PERFORMANCE_HOURS + ) + }) + + test('accepts a custom integer hour range', () => { + assert.equal(toQuery({ ...DEFAULT_FILTERS, hours: '37' }).hours, 37) + assert.equal( + toModelQuery({ ...DEFAULT_MODEL_FILTERS, hours: '37' }).hours, + 37 + ) + }) + + test('normalizes custom hours to the supported query window', () => { + assert.equal(normalizePerformanceHours('0'), DEFAULT_PERFORMANCE_HOURS) + assert.equal(normalizePerformanceHours('-5'), DEFAULT_PERFORMANCE_HOURS) + assert.equal(normalizePerformanceHours('12.8'), 12) + assert.equal(normalizePerformanceHours('999'), MAX_PERFORMANCE_HOURS) + assert.equal( + normalizeFilters({ ...DEFAULT_FILTERS, hours: '999' }).hours, + String(MAX_PERFORMANCE_HOURS) + ) + }) +}) diff --git a/web/default/src/features/smart-ops/lib/filters.ts b/web/default/src/features/smart-ops/lib/filters.ts new file mode 100644 index 00000000..6e453b53 --- /dev/null +++ b/web/default/src/features/smart-ops/lib/filters.ts @@ -0,0 +1,83 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import type { ChannelPerformanceQuery, ModelPerformanceQuery } from '../types' + +export type FilterDraft = { + hours: string + channelId: string + model: string + group: string +} + +export const DEFAULT_PERFORMANCE_HOURS = 1 +export const MAX_PERFORMANCE_HOURS = 168 + +export const DEFAULT_FILTERS: FilterDraft = { + hours: String(DEFAULT_PERFORMANCE_HOURS), + channelId: '', + model: '', + group: '', +} + +export function normalizePerformanceHours(value: string): number { + const parsed = Number(value) + if (!Number.isFinite(parsed)) return DEFAULT_PERFORMANCE_HOURS + return Math.min( + Math.max(Math.trunc(parsed), DEFAULT_PERFORMANCE_HOURS), + MAX_PERFORMANCE_HOURS + ) +} + +export function normalizeFilters(filters: T): T { + return { + ...filters, + hours: String(normalizePerformanceHours(filters.hours)), + } as T +} + +export function toQuery(filters: FilterDraft): ChannelPerformanceQuery { + const channelId = Number.parseInt(filters.channelId, 10) + return { + hours: normalizePerformanceHours(filters.hours), + channelId: + Number.isFinite(channelId) && channelId > 0 ? channelId : undefined, + model: filters.model.trim() || undefined, + group: filters.group.trim() || undefined, + } +} + +export type ModelFilterDraft = { + hours: string + model: string + group: string +} + +export const DEFAULT_MODEL_FILTERS: ModelFilterDraft = { + hours: String(DEFAULT_PERFORMANCE_HOURS), + model: '', + group: '', +} + +export function toModelQuery(filters: ModelFilterDraft): ModelPerformanceQuery { + return { + hours: normalizePerformanceHours(filters.hours), + model: filters.model.trim() || undefined, + group: filters.group.trim() || undefined, + } +} diff --git a/web/default/src/features/smart-ops/lib/format.test.ts b/web/default/src/features/smart-ops/lib/format.test.ts new file mode 100644 index 00000000..0fd45386 --- /dev/null +++ b/web/default/src/features/smart-ops/lib/format.test.ts @@ -0,0 +1,33 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import assert from 'node:assert/strict' +import { describe, test } from 'node:test' +import { formatLegacyLatency } from './format' + +describe('formatLegacyLatency', () => { + test('keeps a recorded zero-second average distinct from missing data', () => { + assert.equal(formatLegacyLatency(0), '0ms') + assert.equal(formatLegacyLatency(null), '—') + }) + + test('uses the shared latency format for positive values', () => { + assert.equal(formatLegacyLatency(500), '500ms') + assert.equal(formatLegacyLatency(1_500), '1.50s') + }) +}) diff --git a/web/default/src/features/smart-ops/lib/format.ts b/web/default/src/features/smart-ops/lib/format.ts new file mode 100644 index 00000000..d4284678 --- /dev/null +++ b/web/default/src/features/smart-ops/lib/format.ts @@ -0,0 +1,33 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import { formatLatency } from '@/features/performance-metrics/lib/format' + +export function formatLegacyLatency( + milliseconds: number | null | undefined +): string { + if ( + milliseconds == null || + !Number.isFinite(milliseconds) || + milliseconds < 0 + ) { + return '—' + } + if (milliseconds === 0) return '0ms' + return formatLatency(milliseconds) +} diff --git a/web/default/src/features/smart-ops/lib/model-sort.test.ts b/web/default/src/features/smart-ops/lib/model-sort.test.ts new file mode 100644 index 00000000..88bc7283 --- /dev/null +++ b/web/default/src/features/smart-ops/lib/model-sort.test.ts @@ -0,0 +1,142 @@ +import assert from 'node:assert/strict' +import { describe, test } from 'node:test' +import type { ModelPerformanceItem } from '../types' +import { + sortModelPerformanceItems, + type ModelPerformanceSortKey, +} from './model-sort' + +function performanceItem( + modelName: string, + values: Partial +): ModelPerformanceItem { + return { + model_name: modelName, + channel_count: 1, + observed_count: 0, + consume_log_count: 0, + error_log_count: 0, + consumed_quota: 0, + retry_log_count: 0, + latency_sample_count: 0, + observed_success_rate: null, + avg_logged_latency_ms: null, + avg_tps: null, + last_observed_at: 1, + quality_flags: [], + ...values, + } +} + +const rows = [ + performanceItem('Balanced model', { + channel_count: 2, + observed_count: 20, + error_log_count: 2, + consumed_quota: 800, + observed_success_rate: 90, + avg_logged_latency_ms: 1000, + avg_tps: 60, + retry_log_count: 1, + }), + performanceItem('Problem model', { + channel_count: 1, + observed_count: 5, + error_log_count: 8, + consumed_quota: 200, + observed_success_rate: 40, + avg_logged_latency_ms: 4500, + avg_tps: 10, + retry_log_count: 4, + }), + performanceItem('Unknown metrics', { + observed_count: 10, + error_log_count: 1, + consumed_quota: 0, + retry_log_count: null, + avg_tps: null, + }), +] + +describe('sortModelPerformanceItems', () => { + const cases: Array<{ + key: ModelPerformanceSortKey + asc: string[] + desc: string[] + }> = [ + { + key: 'model_name', + asc: ['Balanced model', 'Problem model', 'Unknown metrics'], + desc: ['Unknown metrics', 'Problem model', 'Balanced model'], + }, + { + key: 'channel_count', + asc: ['Problem model', 'Unknown metrics', 'Balanced model'], + desc: ['Balanced model', 'Problem model', 'Unknown metrics'], + }, + { + key: 'observed_count', + asc: ['Problem model', 'Unknown metrics', 'Balanced model'], + desc: ['Balanced model', 'Unknown metrics', 'Problem model'], + }, + { + key: 'error_log_count', + asc: ['Unknown metrics', 'Balanced model', 'Problem model'], + desc: ['Problem model', 'Balanced model', 'Unknown metrics'], + }, + { + key: 'consumed_quota', + asc: ['Unknown metrics', 'Problem model', 'Balanced model'], + desc: ['Balanced model', 'Problem model', 'Unknown metrics'], + }, + { + key: 'observed_success_rate', + asc: ['Problem model', 'Balanced model', 'Unknown metrics'], + desc: ['Balanced model', 'Problem model', 'Unknown metrics'], + }, + { + key: 'avg_logged_latency_ms', + asc: ['Balanced model', 'Problem model', 'Unknown metrics'], + desc: ['Problem model', 'Balanced model', 'Unknown metrics'], + }, + { + key: 'avg_tps', + asc: ['Problem model', 'Balanced model', 'Unknown metrics'], + desc: ['Balanced model', 'Problem model', 'Unknown metrics'], + }, + { + key: 'retry_log_count', + asc: ['Balanced model', 'Problem model', 'Unknown metrics'], + desc: ['Problem model', 'Balanced model', 'Unknown metrics'], + }, + ] + + for (const { key, asc, desc } of cases) { + test(`sorts ${key} in both directions and keeps missing values last`, () => { + assert.deepEqual( + sortModelPerformanceItems(rows, { key, direction: 'asc' }).map( + (item) => item.model_name + ), + asc + ) + assert.deepEqual( + sortModelPerformanceItems(rows, { key, direction: 'desc' }).map( + (item) => item.model_name + ), + desc + ) + }) + } + + test('does not mutate the API result order', () => { + const originalOrder = rows.map((item) => item.model_name) + sortModelPerformanceItems(rows, { + key: 'error_log_count', + direction: 'desc', + }) + assert.deepEqual( + rows.map((item) => item.model_name), + originalOrder + ) + }) +}) diff --git a/web/default/src/features/smart-ops/lib/model-sort.ts b/web/default/src/features/smart-ops/lib/model-sort.ts new file mode 100644 index 00000000..197bf081 --- /dev/null +++ b/web/default/src/features/smart-ops/lib/model-sort.ts @@ -0,0 +1,55 @@ +import type { ModelPerformanceItem } from '../types' + +export type ModelPerformanceSortKey = + | 'model_name' + | 'channel_count' + | 'observed_count' + | 'error_log_count' + | 'consumed_quota' + | 'observed_success_rate' + | 'avg_logged_latency_ms' + | 'avg_tps' + | 'retry_log_count' + +export type ModelPerformanceSortState = { + key: ModelPerformanceSortKey + direction: 'asc' | 'desc' +} | null + +function sortableValue( + item: ModelPerformanceItem, + key: ModelPerformanceSortKey +): number | string | null { + const value = item[key] + if (key === 'model_name') return typeof value === 'string' ? value : null + return typeof value === 'number' && Number.isFinite(value) ? value : null +} + +export function sortModelPerformanceItems( + items: readonly ModelPerformanceItem[], + sortState: ModelPerformanceSortState, + locale?: string +) { + if (!sortState) return items + + return items + .map((item, originalIndex) => ({ item, originalIndex })) + .sort((left, right) => { + const leftValue = sortableValue(left.item, sortState.key) + const rightValue = sortableValue(right.item, sortState.key) + + if (leftValue == null && rightValue == null) { + return left.originalIndex - right.originalIndex + } + if (leftValue == null) return 1 + if (rightValue == null) return -1 + + const comparison = + typeof leftValue === 'string' && typeof rightValue === 'string' + ? leftValue.localeCompare(rightValue, locale) + : Number(leftValue) - Number(rightValue) + if (comparison === 0) return left.originalIndex - right.originalIndex + return sortState.direction === 'asc' ? comparison : -comparison + }) + .map(({ item }) => item) +} diff --git a/web/default/src/features/smart-ops/lib/sort.test.ts b/web/default/src/features/smart-ops/lib/sort.test.ts new file mode 100644 index 00000000..4369c615 --- /dev/null +++ b/web/default/src/features/smart-ops/lib/sort.test.ts @@ -0,0 +1,156 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import assert from 'node:assert/strict' +import { describe, test } from 'node:test' +import type { ChannelPerformanceItem } from '../types' +import { + sortChannelPerformanceItems, + type ChannelPerformanceSortKey, +} from './sort' + +function performanceItem( + channelName: string, + values: Partial +): ChannelPerformanceItem { + return { + channel_id: values.channel_id ?? 1, + channel_name: channelName, + channel_type: 1, + channel_status: 1, + model_name: 'test-model', + effective_group: 'default', + observed_count: 0, + consume_log_count: 0, + error_log_count: 0, + consumed_quota: 0, + retry_log_count: 0, + latency_sample_count: 0, + observed_success_rate: null, + avg_logged_latency_ms: null, + last_observed_at: 1, + probe_latency_ms: null, + probe_test_time: null, + quality_flags: [], + ...values, + avg_tps: values.avg_tps ?? null, + } +} + +const rows = [ + performanceItem('Balanced', { + channel_id: 1, + observed_count: 20, + error_log_count: 2, + consumed_quota: 800, + observed_success_rate: 90, + avg_logged_latency_ms: 1000, + retry_log_count: 1, + probe_latency_ms: 80, + }), + performanceItem('Problematic', { + channel_id: 2, + observed_count: 5, + error_log_count: 8, + consumed_quota: 200, + observed_success_rate: 40, + avg_logged_latency_ms: 4500, + retry_log_count: 4, + probe_latency_ms: 900, + }), + performanceItem('Unknown metrics', { + channel_id: 3, + observed_count: 10, + error_log_count: 1, + consumed_quota: 0, + retry_log_count: null, + }), +] + +describe('sortChannelPerformanceItems', () => { + const cases: Array<{ + key: ChannelPerformanceSortKey + asc: string[] + desc: string[] + }> = [ + { + key: 'observed_count', + asc: ['Problematic', 'Unknown metrics', 'Balanced'], + desc: ['Balanced', 'Unknown metrics', 'Problematic'], + }, + { + key: 'error_log_count', + asc: ['Unknown metrics', 'Balanced', 'Problematic'], + desc: ['Problematic', 'Balanced', 'Unknown metrics'], + }, + { + key: 'consumed_quota', + asc: ['Unknown metrics', 'Problematic', 'Balanced'], + desc: ['Balanced', 'Problematic', 'Unknown metrics'], + }, + { + key: 'observed_success_rate', + asc: ['Problematic', 'Balanced', 'Unknown metrics'], + desc: ['Balanced', 'Problematic', 'Unknown metrics'], + }, + { + key: 'avg_logged_latency_ms', + asc: ['Balanced', 'Problematic', 'Unknown metrics'], + desc: ['Problematic', 'Balanced', 'Unknown metrics'], + }, + { + key: 'retry_log_count', + asc: ['Balanced', 'Problematic', 'Unknown metrics'], + desc: ['Problematic', 'Balanced', 'Unknown metrics'], + }, + { + key: 'probe_latency_ms', + asc: ['Balanced', 'Problematic', 'Unknown metrics'], + desc: ['Problematic', 'Balanced', 'Unknown metrics'], + }, + ] + + for (const { key, asc, desc } of cases) { + test(`sorts ${key} in both directions and keeps missing values last`, () => { + assert.deepEqual( + sortChannelPerformanceItems(rows, { key, direction: 'asc' }).map( + (item) => item.channel_name + ), + asc + ) + assert.deepEqual( + sortChannelPerformanceItems(rows, { key, direction: 'desc' }).map( + (item) => item.channel_name + ), + desc + ) + }) + } + + test('does not mutate the API result order', () => { + const originalOrder = rows.map((item) => item.channel_name) + sortChannelPerformanceItems(rows, { + key: 'error_log_count', + direction: 'desc', + }) + assert.deepEqual( + rows.map((item) => item.channel_name), + originalOrder + ) + }) +}) diff --git a/web/default/src/features/smart-ops/lib/sort.ts b/web/default/src/features/smart-ops/lib/sort.ts new file mode 100644 index 00000000..62f3a631 --- /dev/null +++ b/web/default/src/features/smart-ops/lib/sort.ts @@ -0,0 +1,66 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import type { ChannelPerformanceItem } from '../types' + +export type ChannelPerformanceSortKey = + | 'observed_count' + | 'error_log_count' + | 'consumed_quota' + | 'observed_success_rate' + | 'avg_logged_latency_ms' + | 'retry_log_count' + | 'probe_latency_ms' + +export type ChannelPerformanceSortState = { + key: ChannelPerformanceSortKey + direction: 'asc' | 'desc' +} | null + +function sortableMetric( + item: ChannelPerformanceItem, + key: ChannelPerformanceSortKey +) { + const value = item[key] + return typeof value === 'number' && Number.isFinite(value) ? value : null +} + +export function sortChannelPerformanceItems( + items: readonly ChannelPerformanceItem[], + sortState: ChannelPerformanceSortState +) { + if (!sortState) return items + + return items + .map((item, originalIndex) => ({ item, originalIndex })) + .sort((left, right) => { + const leftValue = sortableMetric(left.item, sortState.key) + const rightValue = sortableMetric(right.item, sortState.key) + + if (leftValue == null && rightValue == null) { + return left.originalIndex - right.originalIndex + } + if (leftValue == null) return 1 + if (rightValue == null) return -1 + + const comparison = leftValue - rightValue + if (comparison === 0) return left.originalIndex - right.originalIndex + return sortState.direction === 'asc' ? comparison : -comparison + }) + .map(({ item }) => item) +} diff --git a/web/default/src/features/smart-ops/model-performance.test.tsx b/web/default/src/features/smart-ops/model-performance.test.tsx new file mode 100644 index 00000000..4f11647f --- /dev/null +++ b/web/default/src/features/smart-ops/model-performance.test.tsx @@ -0,0 +1,685 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import { act } from 'react' +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { createReactTestEnvironment } from '@/test/react' +import { waitFor } from '@testing-library/react' +import assert from 'node:assert/strict' +import { after, before, describe, test } from 'node:test' +import { api } from '@/lib/api' +import { formatQuota } from '@/lib/format' +import { formatThroughput } from '@/features/performance-metrics/lib/format' +import type { + PerformanceAggregate, + PerformanceGroup, +} from '@/features/performance-metrics/types' +import { + buildLatencyChartData, + buildUptimeChartData, + downsampleUptimeSeries, + getUptimeAxisDomain, +} from '@/features/pricing/components/model-details-chart-utils' +import { ModelPerformanceDetailsContent } from '@/features/pricing/components/model-details-performance' +import { UptimeSparkline } from '@/features/pricing/components/model-details-uptime-sparkline' +import { + ModelPerformance, + ModelPerformanceDetailGrid, +} from './model-performance' +import type { ModelPerformanceData } from './types' + +const testEnv = createReactTestEnvironment() + +before(() => testEnv.setup()) + +after(() => testEnv.teardown()) + +const modelPerformanceData: ModelPerformanceData = { + storage_mode: 'legacy_log', + source: 'log_db', + partial: true, + quality_flags: ['legacy'], + time_range: { start_at: 1, end_at: 3601, hours: 1 }, + throughput: { + collection_state: 'available', + coverage: { + requested_start_at: 1, + requested_end_at: 3601, + bucket_start_at: 3600, + bucket_end_at: 3601, + bucket_seconds: 3600, + granularity_state: 'unknown', + approximate: true, + }, + }, + summary: { + model_count: 2, + channel_count: 3, + observed_count: 15, + consume_log_count: 12, + error_log_count: 3, + consumed_quota: 1200000, + retry_log_count: 2, + latency_sample_count: 12, + observed_success_rate: 80, + avg_logged_latency_ms: 1200, + last_observed_at: 3600, + }, + items: [ + { + model_name: 'alpha', + channel_count: 2, + observed_count: 10, + consume_log_count: 8, + error_log_count: 2, + consumed_quota: 800000, + retry_log_count: 1, + latency_sample_count: 8, + observed_success_rate: 80, + avg_logged_latency_ms: 1000, + avg_tps: 48.5, + last_observed_at: 3600, + quality_flags: ['legacy'], + }, + { + model_name: 'beta', + channel_count: 1, + observed_count: 5, + consume_log_count: 4, + error_log_count: 1, + consumed_quota: 400000, + retry_log_count: 1, + latency_sample_count: 4, + observed_success_rate: 80, + avg_logged_latency_ms: 1600, + avg_tps: 12.25, + last_observed_at: 3500, + quality_flags: ['legacy'], + }, + ], + truncated: false, + generated_at: 3601, +} + +function renderModelPerformance() { + const queryClient = new QueryClient({ + defaultOptions: { + queries: { retry: false }, + mutations: { retry: false }, + }, + }) + return { + queryClient, + render: testEnv.render( + + + + ), + } +} + +describe('ModelPerformance manual queries', () => { + test('does not query on mount and sends the default hour range only after Apply filters', async () => { + const originalGet = api.get + let requestCount = 0 + let requestParams: Record | undefined + api.get = (async (_url, config) => { + requestCount += 1 + requestParams = config?.params as Record + return { data: { success: true, data: modelPerformanceData } } + }) as typeof api.get + + const { queryClient, render } = renderModelPerformance() + const view = await render + + try { + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 0)) + }) + assert.equal(requestCount, 0, 'mounting must not query model logs') + + const applyButton = Array.from( + view.container.querySelectorAll('button') + ).find((candidate) => candidate.textContent?.trim() === 'Apply filters') + assert.ok(applyButton, 'expected Apply filters button') + await view.click(applyButton) + + await waitFor(() => assert.equal(requestCount, 1)) + assert.equal(requestParams?.hours, 1) + assert.equal(requestParams?.limit, 200) + assert.equal(requestParams?.model, undefined) + assert.equal(requestParams?.group, undefined) + } finally { + api.get = originalGet + queryClient.clear() + await view.unmount() + } + }) + + test('renders every returned model with errors, consumed quota, and throughput', async () => { + const originalGet = api.get + api.get = (async () => ({ + data: { success: true, data: modelPerformanceData }, + })) as typeof api.get + + const { queryClient, render } = renderModelPerformance() + const view = await render + + try { + const applyButton = Array.from( + view.container.querySelectorAll('button') + ).find((candidate) => candidate.textContent?.trim() === 'Apply filters') + assert.ok(applyButton, 'expected Apply filters button') + await view.click(applyButton) + + await waitFor(() => { + assert.equal(view.container.querySelectorAll('tbody tr').length, 2) + }) + + assert.ok(view.container.textContent?.includes('alpha')) + assert.ok(view.container.textContent?.includes('beta')) + const errorCardLabel = Array.from( + view.container.querySelectorAll('[data-slot="card-description"]') + ).find((node) => node.textContent?.trim() === 'Recorded error logs') + assert.ok(errorCardLabel, 'expected recorded error summary card') + assert.equal( + errorCardLabel.parentElement?.querySelector('[data-slot="card-title"]') + ?.textContent, + '3' + ) + const headers = Array.from(view.container.querySelectorAll('thead th')) + const errorColumnIndex = headers.findIndex( + (header) => header.textContent?.trim() === 'Recorded error logs' + ) + assert.ok(errorColumnIndex >= 0, 'expected error count column') + const rows = view.container.querySelectorAll('tbody tr') + assert.equal( + rows + .item(0) + .querySelectorAll('td') + .item(errorColumnIndex) + .textContent?.trim(), + '2' + ) + assert.equal( + rows + .item(1) + .querySelectorAll('td') + .item(errorColumnIndex) + .textContent?.trim(), + '1' + ) + const consumedQuotaColumnIndex = headers.findIndex( + (header) => header.textContent?.trim() === 'Consumed quota' + ) + assert.ok(consumedQuotaColumnIndex >= 0, 'expected consumed quota column') + assert.equal( + rows + .item(0) + .querySelectorAll('td') + .item(consumedQuotaColumnIndex) + .textContent?.trim(), + formatQuota(800000) + ) + const throughputColumnIndex = headers.findIndex( + (header) => header.textContent?.trim() === 'Throughput' + ) + assert.ok(throughputColumnIndex >= 0, 'expected throughput column') + assert.equal( + rows + .item(0) + .querySelectorAll('td') + .item(throughputColumnIndex) + .textContent?.trim(), + formatThroughput(48.5) + ) + } finally { + api.get = originalGet + queryClient.clear() + await view.unmount() + } + }) + + test('exposes a sort menu for each model performance metric', async () => { + const originalGet = api.get + api.get = (async () => ({ + data: { success: true, data: modelPerformanceData }, + })) as typeof api.get + + const { queryClient, render } = renderModelPerformance() + const view = await render + + try { + const applyButton = Array.from( + view.container.querySelectorAll('button') + ).find((candidate) => candidate.textContent?.trim() === 'Apply filters') + assert.ok(applyButton, 'expected Apply filters button') + await view.click(applyButton) + + const sortableLabels = [ + 'Model', + 'Channels', + 'Observed model log events', + 'Recorded error logs', + 'Consumed quota', + 'Estimated success rate', + 'Logged latency', + 'Throughput', + 'Retries', + ] + await waitFor(() => { + const buttons = Array.from( + view.container.querySelectorAll('thead button') + ) + for (const label of sortableLabels) { + assert.ok( + buttons.some((button) => button.textContent?.trim() === label), + `expected sortable ${label} header` + ) + } + }) + } finally { + api.get = originalGet + queryClient.clear() + await view.unmount() + } + }) + + test('distinguishes disabled, empty, and failed throughput collection states', async () => { + const originalGet = api.get + const scenarios = [ + { + state: 'collection_disabled' as const, + flag: 'throughput_collection_disabled', + title: 'Performance metrics collection is disabled', + }, + { + state: 'no_samples' as const, + flag: 'throughput_no_samples', + title: 'No throughput samples', + }, + { + state: 'query_failed' as const, + flag: 'throughput_query_failed', + title: 'Throughput query failed', + }, + ] + + try { + for (const scenario of scenarios) { + api.get = (async () => ({ + data: { + success: true, + data: { + ...modelPerformanceData, + quality_flags: [scenario.flag], + throughput: { + ...modelPerformanceData.throughput, + collection_state: scenario.state, + }, + }, + }, + })) as typeof api.get + + const { queryClient, render } = renderModelPerformance() + const view = await render + try { + const applyButton = Array.from( + view.container.querySelectorAll('button') + ).find( + (candidate) => candidate.textContent?.trim() === 'Apply filters' + ) + assert.ok(applyButton, 'expected Apply filters button') + await view.click(applyButton) + await waitFor(() => { + assert.ok(view.container.textContent?.includes(scenario.title)) + }) + } finally { + queryClient.clear() + await view.unmount() + } + } + } finally { + api.get = originalGet + } + }) + + test('does not claim a precise bucket size when historical granularity is unknown', async () => { + const originalGet = api.get + api.get = (async () => ({ + data: { + success: true, + data: { + ...modelPerformanceData, + quality_flags: ['throughput_window_approximate'], + }, + }, + })) as typeof api.get + + const { queryClient, render } = renderModelPerformance() + const view = await render + + try { + const applyButton = Array.from( + view.container.querySelectorAll('button') + ).find((candidate) => candidate.textContent?.trim() === 'Apply filters') + assert.ok(applyButton, 'expected Apply filters button') + await view.click(applyButton) + + await waitFor(() => { + const text = view.container.textContent ?? '' + assert.ok(text.includes('Bucket granularity is unknown or mixed')) + assert.ok(!text.includes('60 minute buckets')) + }) + } finally { + api.get = originalGet + queryClient.clear() + await view.unmount() + } + }) + + test('loads 24-hour model performance only after opening details', async () => { + const originalGet = api.get + let detailRequests = 0 + api.get = (async (url, config) => { + if (url === '/api/smart-ops/model-performance/detail') { + detailRequests += 1 + assert.equal(config?.params?.model, 'alpha') + return { + data: { + success: true, + data: { + model_name: 'alpha', + series_schema: 'test', + collection_state: 'available', + coverage: { + requested_start_at: 1, + requested_end_at: 3601, + bucket_start_at: 3600, + bucket_end_at: 3601, + bucket_seconds: 3600, + granularity_state: 'unknown', + approximate: true, + }, + summary: { + avg_ttft_ms: 320, + avg_latency_ms: 1200, + success_rate: 99.5, + avg_tps: 48.5, + series: [ + { + ts: 3600, + avg_ttft_ms: 320, + avg_latency_ms: 1200, + success_rate: 99.5, + avg_tps: 48.5, + }, + ], + }, + groups: [ + { + group: 'default', + avg_ttft_ms: 320, + avg_latency_ms: 1200, + success_rate: 99.5, + avg_tps: 48.5, + series: [ + { + ts: 3600, + avg_ttft_ms: 320, + avg_latency_ms: 1200, + success_rate: 99.5, + avg_tps: 48.5, + }, + ], + }, + ], + }, + }, + } + } + return { data: { success: true, data: modelPerformanceData } } + }) as typeof api.get + + const { queryClient, render } = renderModelPerformance() + const view = await render + + try { + const applyButton = Array.from( + view.container.querySelectorAll('button') + ).find((candidate) => candidate.textContent?.trim() === 'Apply filters') + assert.ok(applyButton, 'expected Apply filters button') + await view.click(applyButton) + await waitFor(() => { + assert.equal(view.container.querySelectorAll('tbody tr').length, 2) + }) + assert.equal(detailRequests, 0, 'list loading must not fetch detail data') + + const detailsButton = Array.from( + view.container.querySelectorAll('tbody button') + ).find((candidate) => candidate.textContent?.trim() === 'Details') + assert.ok(detailsButton, 'expected model details button') + await view.click(detailsButton) + + await waitFor(() => { + assert.equal(detailRequests, 1) + }) + } finally { + api.get = originalGet + queryClient.clear() + await view.unmount() + } + }) + + test('renders per-group performance and 24-hour trend sections', async () => { + const groups: PerformanceGroup[] = [ + { + group: 'default', + avg_ttft_ms: 320, + avg_latency_ms: 1200, + success_rate: 99.5, + avg_tps: 48.5, + series: [ + { + ts: 3600, + avg_ttft_ms: 320, + avg_latency_ms: 1200, + success_rate: 99.5, + avg_tps: 48.5, + }, + ], + }, + ] + const summary: PerformanceAggregate = { + avg_ttft_ms: 320, + avg_latency_ms: 1200, + success_rate: 99.5, + avg_tps: 48.5, + series: groups[0].series, + } + const view = await testEnv.render( + + ) + + try { + const detailText = view.container.textContent ?? '' + assert.ok(detailText.includes('Per-group performance')) + assert.ok(detailText.includes('Latency trend (last 24h)')) + assert.ok(detailText.includes('Availability (last 24h)')) + assert.ok(detailText.includes('default')) + assert.ok(detailText.includes(formatThroughput(48.5))) + } finally { + await view.unmount() + } + }) + + test('prevents the model detail summary grid from collapsing in the sheet', async () => { + const view = await testEnv.render( +
+ +
+ ) + + try { + const detailGrid = view.container.querySelector('dl') + assert.ok(detailGrid, 'expected model detail grid') + assert.ok( + detailGrid.classList.contains('shrink-0'), + 'model detail grid must not collapse inside a constrained flex sheet' + ) + } finally { + await view.unmount() + } + }) + + test('uses the backend weighted model aggregate instead of averaging groups', async () => { + const groups: PerformanceGroup[] = [ + { + group: 'small', + avg_ttft_ms: 1000, + avg_latency_ms: 1000, + success_rate: 0, + avg_tps: 10, + series: [ + { + ts: 3600, + avg_ttft_ms: 1000, + avg_latency_ms: 1000, + success_rate: 0, + avg_tps: 10, + }, + ], + }, + { + group: 'large', + avg_ttft_ms: 50, + avg_latency_ms: 100, + success_rate: 100, + avg_tps: 100, + series: [ + { + ts: 3600, + avg_ttft_ms: 50, + avg_latency_ms: 100, + success_rate: 100, + avg_tps: 100, + }, + ], + }, + ] + const summary: PerformanceAggregate = { + avg_ttft_ms: 50, + avg_latency_ms: 109, + success_rate: 99, + avg_tps: 91.743, + series: [], + } + const view = await testEnv.render( + + ) + + try { + const detailText = view.container.textContent ?? '' + assert.ok(detailText.includes('99.00%')) + assert.ok(detailText.includes(formatThroughput(91.743))) + assert.ok(!detailText.includes('50.00%')) + } finally { + await view.unmount() + } + }) + + test('expands the uptime axis to include severe availability failures', () => { + assert.deepEqual( + getUptimeAxisDomain([ + { + date: new Date(3600 * 1000).toISOString(), + uptime_pct: 4.11, + incidents: 1, + outage_minutes: 0, + }, + { + date: new Date(7200 * 1000).toISOString(), + uptime_pct: 99.5, + incidents: 1, + outage_minutes: 0, + }, + ]), + { min: 0, max: 100 } + ) + }) + + test('keeps minute-level chart categories unique', () => { + const first = new Date('2026-08-18T10:01:00.000Z').toISOString() + const second = new Date('2026-08-18T10:02:00.000Z').toISOString() + + const latency = buildLatencyChartData([ + { timestamp: first, group: 'default', ttft_ms: 100 }, + { timestamp: second, group: 'default', ttft_ms: 120 }, + ]) + const uptime = buildUptimeChartData([ + { date: first, uptime_pct: 99, incidents: 1, outage_minutes: 0 }, + { date: second, uptime_pct: 100, incidents: 0, outage_minutes: 0 }, + ]) + + assert.notEqual(latency[0].time, latency[1].time) + assert.notEqual(uptime[0].date, uptime[1].date) + }) + + test('downsamples dense uptime series while preserving the worst point in each window', () => { + const series = Array.from({ length: 100 }, (_, index) => ({ + date: new Date((index + 1) * 60_000).toISOString(), + uptime_pct: index === 49 ? 12.5 : 100, + incidents: index === 49 ? 1 : 0, + outage_minutes: 0, + })) + + const sampled = downsampleUptimeSeries(series, 32) + + assert.equal(sampled.length, 32) + assert.ok(sampled.some((point) => point.uptime_pct === 12.5)) + }) + + test('uses the weighted overall uptime supplied by the backend', async () => { + const view = await testEnv.render( + + ) + + try { + assert.ok(view.container.textContent?.includes('99.0%')) + assert.ok(!view.container.textContent?.includes('50.0%')) + } finally { + await view.unmount() + } + }) +}) diff --git a/web/default/src/features/smart-ops/model-performance.tsx b/web/default/src/features/smart-ops/model-performance.tsx new file mode 100644 index 00000000..173acaa1 --- /dev/null +++ b/web/default/src/features/smart-ops/model-performance.tsx @@ -0,0 +1,1020 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import { useMemo, useState } from 'react' +import { useMutation } from '@tanstack/react-query' +import type { TFunction } from 'i18next' +import { + AlertTriangle, + ArrowDown, + ArrowUp, + Bot, + ChevronsUpDown, + Clock3, + DatabaseZap, + HeartPulse, + RefreshCw, + Search, + ShieldCheck, +} from 'lucide-react' +import { useTranslation } from 'react-i18next' +import { formatQuota } from '@/lib/format' +import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert' +import { Badge } from '@/components/ui/badge' +import { Button } from '@/components/ui/button' +import { + Card, + CardAction, + CardDescription, + CardHeader, + CardTitle, +} from '@/components/ui/card' +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuGroup, + DropdownMenuItem, + DropdownMenuTrigger, +} from '@/components/ui/dropdown-menu' +import { + Empty, + EmptyDescription, + EmptyHeader, + EmptyMedia, + EmptyTitle, +} from '@/components/ui/empty' +import { Field, FieldGroup, FieldLabel } from '@/components/ui/field' +import { Input } from '@/components/ui/input' +import { + Sheet, + SheetContent, + SheetDescription, + SheetHeader, + SheetTitle, +} from '@/components/ui/sheet' +import { Skeleton } from '@/components/ui/skeleton' +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@/components/ui/table' +import { ErrorState } from '@/components/error-state' +import { SectionPageLayout } from '@/components/layout' +import { + formatThroughput, + formatUptimePct, +} from '@/features/performance-metrics/lib/format' +import type { PerformanceCoverage } from '@/features/performance-metrics/types' +import { ModelPerformanceDetailsContent } from '@/features/pricing/components/model-details-performance' +import { getModelPerformance, getModelPerformanceDetail } from './api' +import { + DEFAULT_MODEL_FILTERS, + MAX_PERFORMANCE_HOURS, + normalizeFilters, + toModelQuery, + type ModelFilterDraft, +} from './lib/filters' +import { formatLegacyLatency } from './lib/format' +import { + sortModelPerformanceItems, + type ModelPerformanceSortKey, + type ModelPerformanceSortState, +} from './lib/model-sort' +import type { ModelPerformanceItem } from './types' + +const priorityQualityFlags = [ + 'error_logs_disabled', + 'consume_logs_disabled', + 'retry_backfill_pending', + 'retry_metrics_unavailable', + 'throughput_collection_disabled', + 'throughput_no_samples', + 'throughput_query_failed', + 'throughput_window_approximate', + 'time_range_clamped', + 'truncated', + 'incomplete_errors_possible', +] + +function getVisibleQualityFlags(flags: string[]) { + const flagSet = new Set(flags) + const priorityFlags = priorityQualityFlags.filter((flag) => flagSet.has(flag)) + const otherFlags = flags.filter( + (flag) => !priorityQualityFlags.includes(flag) + ) + return [ + ...priorityFlags, + ...otherFlags.slice(0, Math.max(0, 4 - priorityFlags.length)), + ] +} + +export function ModelPerformance() { + const { t, i18n } = useTranslation() + const [draft, setDraft] = useState(DEFAULT_MODEL_FILTERS) + const [filters, setFilters] = useState( + DEFAULT_MODEL_FILTERS + ) + const [selected, setSelected] = useState(null) + const [sortState, setSortState] = useState(null) + const queryParams = useMemo(() => toModelQuery(filters), [filters]) + const numberFormatter = useMemo( + () => new Intl.NumberFormat(i18n.language), + [i18n.language] + ) + + const performanceQuery = useMutation({ + mutationKey: ['smart-ops', 'model-performance'], + mutationFn: getModelPerformance, + retry: false, + }) + const detailQuery = useMutation({ + mutationKey: ['smart-ops', 'model-performance-detail'], + mutationFn: getModelPerformanceDetail, + retry: false, + }) + + const data = performanceQuery.data + const sortedItems = useMemo( + () => + sortModelPerformanceItems(data?.items ?? [], sortState, i18n.language), + [data?.items, i18n.language, sortState] + ) + const latestObservedAt = data?.summary.last_observed_at ?? 0 + const retryBackfillPending = + data?.quality_flags.includes('retry_backfill_pending') ?? false + const retryMetricsUnavailable = + data?.quality_flags.includes('retry_metrics_unavailable') ?? false + const errorLogsDisabled = + data?.quality_flags.includes('error_logs_disabled') ?? false + const throughputCollectionDisabled = + data?.quality_flags.includes('throughput_collection_disabled') ?? false + const throughputNoSamples = + data?.quality_flags.includes('throughput_no_samples') ?? false + const throughputQueryFailed = + data?.quality_flags.includes('throughput_query_failed') ?? false + const throughputPartial = + data?.quality_flags.includes('throughput_partial') ?? false + const throughputWindowApproximate = + data?.quality_flags.includes('throughput_window_approximate') ?? false + + const openDetails = (item: ModelPerformanceItem) => { + detailQuery.reset() + setSelected(item) + detailQuery.mutate(item.model_name) + } + + const formatTimestamp = (timestamp: number | null | undefined) => { + if (!timestamp) return '—' + return new Intl.DateTimeFormat(i18n.language, { + dateStyle: 'medium', + timeStyle: 'medium', + }).format(new Date(timestamp * 1000)) + } + + const qualityLabel = (flag: string) => { + const labels: Record = { + legacy: t('Legacy log'), + partial: t('Partial data'), + weak_correlation: t('Weak correlation'), + heuristic_outcome: t('Heuristic outcome'), + coarse_latency: t('Coarse latency'), + non_attempt_logs_excluded: t('Known non-attempt logs excluded'), + time_range_clamped: t('Time range was limited'), + truncated: t('Results truncated'), + retry_backfill_pending: t('Retry marker backfill pending'), + retry_metrics_unavailable: t('Retry metrics unavailable'), + consume_logs_disabled: t('Consumption logs disabled'), + error_logs_disabled: t('Error logs disabled'), + incomplete_errors_possible: t('Errors may be incomplete'), + throughput_collection_disabled: t( + 'Performance metrics collection is disabled' + ), + throughput_no_samples: t('No throughput samples'), + throughput_query_failed: t('Throughput query failed'), + throughput_window_approximate: t('Bucket-level estimate'), + } + return labels[flag] ?? flag + } + + return ( + + + + {t('Model performance')} + {t('Read only')} + {t('Legacy log')} + + + + + + +
+ + + + {(retryBackfillPending || retryMetricsUnavailable) && ( + + + )} + + {errorLogsDisabled && ( + + + )} + + {throughputCollectionDisabled && ( + + + )} + + {throughputNoSamples && ( + + + )} + + {throughputQueryFailed && ( + + + )} + + {throughputPartial && ( + + + )} + + {throughputWindowApproximate && data?.throughput?.coverage && ( + + + )} + +
{ + event.preventDefault() + const normalizedDraft = normalizeFilters(draft) + const nextQuery = toModelQuery(normalizedDraft) + setDraft(normalizedDraft) + setFilters(normalizedDraft) + performanceQuery.mutate(nextQuery) + }} + > + + + + {t('Time range')} ({t('hours')}, 1–{MAX_PERFORMANCE_HOURS}) + + + setDraft((current) => ({ + ...current, + hours: event.target.value, + })) + } + /> + + + + {t('Model')} + + + setDraft((current) => ({ + ...current, + model: event.target.value, + })) + } + /> + + + + {t('Group')} + + + setDraft((current) => ({ + ...current, + group: event.target.value, + })) + } + /> + + + +
+ +
+ + + + + + +
+ + {performanceQuery.isPending ? ( + + ) : performanceQuery.isError ? ( + performanceQuery.mutate(queryParams)} + /> + ) : performanceQuery.isIdle ? ( + + + + + + {t('Model performance is not loaded automatically')} + + + {t( + 'Select Apply filters or Refresh to query the current time range.' + )} + + + + ) : !data?.items.length ? ( + + + + + {t('No recorded model logs')} + + {t( + 'Try a wider time range or check whether consumption and error logging are enabled.' + )} + + + + ) : ( +
+
+ + {t('Model performance')} + + + {t('{{count}} rows', { count: data.items.length })} + +
+ {getVisibleQualityFlags(data.quality_flags).map((flag) => ( + + {qualityLabel(flag)} + + ))} +
+
+ + + + + + + + + + + + + {t('Last observed')} + {t('Actions')} + + + + {sortedItems.map((item) => ( + openDetails(item)} + > + + {item.model_name} + + + {numberFormatter.format(item.channel_count)} + + + {numberFormatter.format(item.observed_count)} + + + {numberFormatter.format(item.error_log_count)} + + + {formatQuota(item.consumed_quota)} + + + {item.observed_success_rate == null + ? t('N/A') + : formatUptimePct(item.observed_success_rate)} + + + {formatLegacyLatency(item.avg_logged_latency_ms)} + + + {formatThroughput(item.avg_tps ?? Number.NaN)} + + + {item.retry_log_count == null + ? t('N/A') + : numberFormatter.format(item.retry_log_count)} + + + {formatTimestamp(item.last_observed_at)} + + + + + + ))} + +
+ {data.truncated && ( +
+ {t( + 'The table was truncated to {{count}} rows. Summary cards still cover all matching eligible logs.', + { count: data.items.length } + )} +
+ )} +
+ )} + + {data && ( +
+ {t('Generated at {{time}}', { + time: formatTimestamp(data.generated_at), + })} +
+ )} + { + if (!open) { + setSelected(null) + detailQuery.reset() + } + }} + > + + + {t('Model performance details')} + + {selected?.model_name ?? ''} + + + {selected && ( +
+ +
+
+ {t('Data quality')} +
+
+ {selected.quality_flags.map((flag) => ( + + {qualityLabel(flag)} + + ))} +
+
+ + + {detailQuery.isPending ? ( + + ) : detailQuery.isError ? ( + detailQuery.mutate(selected.model_name)} + /> + ) : detailQuery.data ? ( + <> + {detailQuery.data.collection_state === + 'collection_disabled' && ( + + + )} + {detailQuery.data.collection_state === 'no_samples' && ( + + + )} + {detailQuery.data.collection_state === 'partial' && ( + + + )} + {detailQuery.data.coverage?.approximate && ( + + + )} + + + ) : null} +
+ )} + + +
+ + + ) +} + +function SortableModelPerformanceHead({ + sortKey, + title, + sortState, + onSort, +}: { + sortKey: ModelPerformanceSortKey + title: string + sortState: ModelPerformanceSortState + onSort: (sortState: ModelPerformanceSortState) => void +}) { + const { t } = useTranslation() + const direction = sortState?.key === sortKey ? sortState.direction : null + const ariaSort = + direction === 'asc' + ? 'ascending' + : direction === 'desc' + ? 'descending' + : 'none' + + return ( + +
+ + + } + > + {title} + {direction === 'desc' ? ( + + + + onSort({ key: sortKey, direction: 'asc' })} + > + + onSort({ key: sortKey, direction: 'desc' })} + > + + + + +
+ + ) +} + +function MetricCard({ + icon: Icon, + label, + value, + loading, +}: { + icon: typeof DatabaseZap + label: string + value: string + loading: boolean +}) { + return ( + + + {label} + + {loading ? : value} + + +
+
+ + + + ) +} + +function PerformanceTableSkeleton() { + return ( + + ) +} + +function ModelPerformanceDetailSkeleton() { + return ( + + ) +} + +export function ModelPerformanceDetailGrid({ + rows, +}: { + rows: [string, string][] +}) { + return ( +
+ {rows.map(([label, value]) => ( +
+
{label}
+
+ {value} +
+
+ ))} +
+ ) +} + +function formatCoverageDescription( + t: TFunction, + coverage: PerformanceCoverage, + formatTimestamp: (timestamp: number | null | undefined) => string +) { + const values = { + start: formatTimestamp(coverage.bucket_start_at), + end: formatTimestamp(coverage.bucket_end_at), + } + if (coverage.granularity_state === 'known' && coverage.bucket_seconds > 0) { + return t( + 'Throughput coverage: {{start}} – {{end}} ({{minutes}} minute buckets).', + { + ...values, + minutes: Math.max(1, Math.round(coverage.bucket_seconds / 60)), + } + ) + } + return t( + 'Throughput coverage: {{start}} – {{end}}. Bucket granularity is unknown or mixed, so this range is approximate.', + values + ) +} diff --git a/web/default/src/features/smart-ops/types.ts b/web/default/src/features/smart-ops/types.ts new file mode 100644 index 00000000..2e3e26de --- /dev/null +++ b/web/default/src/features/smart-ops/types.ts @@ -0,0 +1,161 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import type { + PerformanceAggregate, + PerformanceCollectionState, + PerformanceCoverage, + PerformanceGroup, +} from '@/features/performance-metrics/types' + +export type ChannelPerformanceQuery = { + hours: number + channelId?: number + model?: string + group?: string +} + +export type ChannelPerformanceItem = { + channel_id: number + channel_name: string + channel_type: number | null + channel_status: number | null + model_name: string + effective_group: string + observed_count: number + consume_log_count: number + error_log_count: number + consumed_quota: number + retry_log_count: number | null + latency_sample_count: number + observed_success_rate: number | null + avg_logged_latency_ms: number | null + avg_tps: number | null + last_observed_at: number + probe_latency_ms: number | null + probe_test_time: number | null + quality_flags: string[] +} + +export type ChannelPerformanceData = { + storage_mode: 'legacy_log' + source: 'log_db' + partial: boolean + quality_flags: string[] + time_range: { + start_at: number + end_at: number + hours: number + } + summary: { + channel_count: number + observed_count: number + consume_log_count: number + error_log_count: number + consumed_quota: number + retry_log_count: number | null + latency_sample_count: number + observed_success_rate: number | null + avg_logged_latency_ms: number | null + last_observed_at: number + } + items: ChannelPerformanceItem[] + truncated: boolean + generated_at: number +} + +export type ChannelPerformanceResponse = { + success: boolean + message?: string + data: ChannelPerformanceData +} + +export type ModelPerformanceQuery = { + hours: number + model?: string + group?: string +} + +export type ModelPerformanceItem = { + model_name: string + channel_count: number + observed_count: number + consume_log_count: number + error_log_count: number + consumed_quota: number + retry_log_count: number | null + latency_sample_count: number + observed_success_rate: number | null + avg_logged_latency_ms: number | null + avg_tps: number | null + last_observed_at: number + quality_flags: string[] +} + +export type ModelPerformanceData = { + storage_mode: 'legacy_log' + source: 'log_db' + partial: boolean + quality_flags: string[] + time_range: { + start_at: number + end_at: number + hours: number + } + summary: { + model_count: number + channel_count: number + observed_count: number + consume_log_count: number + error_log_count: number + consumed_quota: number + retry_log_count: number | null + latency_sample_count: number + observed_success_rate: number | null + avg_logged_latency_ms: number | null + last_observed_at: number + } + throughput?: { + collection_state: PerformanceCollectionState + coverage: PerformanceCoverage + } + items: ModelPerformanceItem[] + truncated: boolean + generated_at: number +} + +export type ModelPerformanceResponse = { + success: boolean + message?: string + data: ModelPerformanceData +} + +export type ModelPerformanceDetailData = { + model_name: string + series_schema?: string + summary?: PerformanceAggregate + collection_state?: PerformanceCollectionState + coverage?: PerformanceCoverage + groups: PerformanceGroup[] +} + +export type ModelPerformanceDetailResponse = { + success: boolean + message?: string + data: ModelPerformanceDetailData +} diff --git a/web/default/src/features/system-settings/maintenance/config.ts b/web/default/src/features/system-settings/maintenance/config.ts index 78277223..e49b9a48 100644 --- a/web/default/src/features/system-settings/maintenance/config.ts +++ b/web/default/src/features/system-settings/maintenance/config.ts @@ -64,6 +64,7 @@ export const SIDEBAR_MODULES_DEFAULT: SidebarModulesAdminConfig = { user: true, setting: true, subscription: true, + smart_ops: true, }, } diff --git a/web/default/src/features/system-settings/maintenance/sidebar-modules-section.tsx b/web/default/src/features/system-settings/maintenance/sidebar-modules-section.tsx index 8c5da598..57cc5e2b 100644 --- a/web/default/src/features/system-settings/maintenance/sidebar-modules-section.tsx +++ b/web/default/src/features/system-settings/maintenance/sidebar-modules-section.tsx @@ -150,6 +150,12 @@ export function SidebarModulesSection({ title: t('Subscription Management'), description: t('Manage subscription plans and pricing.'), }, + smart_ops: { + title: t('Smart Operations'), + description: t( + 'Inspect read-only production performance and operational evidence.' + ), + }, }, } const formDefaults = useMemo(() => config, [config]) diff --git a/web/default/src/hooks/use-sidebar-config.ts b/web/default/src/hooks/use-sidebar-config.ts index 8c013279..68207264 100644 --- a/web/default/src/hooks/use-sidebar-config.ts +++ b/web/default/src/hooks/use-sidebar-config.ts @@ -62,6 +62,7 @@ const DEFAULT_SIDEBAR_MODULES: SidebarModulesAdminConfig = { user: true, setting: true, subscription: true, + smart_ops: true, }, } @@ -113,9 +114,26 @@ const URL_TO_CONFIG_MAP: Record = { '/users': { section: 'admin', module: 'user' }, '/redemption-codes': { section: 'admin', module: 'redemption' }, '/subscriptions': { section: 'admin', module: 'subscription' }, + '/smart-ops': { section: 'admin', module: 'smart_ops' }, + '/smart-ops/production-performance': { + section: 'admin', + module: 'smart_ops', + }, + '/smart-ops/channel-performance': { + section: 'admin', + module: 'smart_ops', + }, + '/smart-ops/model-performance': { + section: 'admin', + module: 'smart_ops', + }, + '/smart-ops/system-info': { + section: 'admin', + module: 'smart_ops', + }, '/system-settings': { section: 'admin', module: 'setting' }, '/system-settings/site': { section: 'admin', module: 'setting' }, - '/system-info': { section: 'admin', module: 'setting' }, + '/system-info': { section: 'admin', module: 'smart_ops' }, } /** diff --git a/web/default/src/hooks/use-sidebar-data.ts b/web/default/src/hooks/use-sidebar-data.ts index c930f8e2..36a6a525 100644 --- a/web/default/src/hooks/use-sidebar-data.ts +++ b/web/default/src/hooks/use-sidebar-data.ts @@ -22,12 +22,12 @@ import { CreditCard, FileText, FlaskConical, + HeartPulse, Key, LayoutDashboard, ListTodo, MessageSquare, Radio, - ServerCog, Settings, Ticket, User, @@ -35,8 +35,6 @@ import { Wallet, } from 'lucide-react' import { useTranslation } from 'react-i18next' -import { useAuthStore } from '@/stores/auth-store' -import { ROLE } from '@/lib/roles' import { type SidebarData } from '@/components/layout/types' /** @@ -47,8 +45,6 @@ import { type SidebarData } from '@/components/layout/types' */ export function useSidebarData(): SidebarData { const { t } = useTranslation() - const user = useAuthStore((state) => state.auth.user) - const isSuperAdmin = user?.role === ROLE.SUPER_ADMIN return { navGroups: [ @@ -146,21 +142,18 @@ export function useSidebarData(): SidebarData { url: '/subscriptions', icon: CreditCard, }, + { + title: t('Smart Operations'), + url: '/smart-ops/channel-performance', + activeUrls: ['/smart-ops'], + icon: HeartPulse, + }, { title: t('System Settings'), url: '/system-settings/site', activeUrls: ['/system-settings'], icon: Settings, }, - ...(isSuperAdmin - ? [ - { - title: t('System Info'), - url: '/system-info', - icon: ServerCog, - }, - ] - : []), ], }, ], diff --git a/web/default/src/i18n/locales/en.json b/web/default/src/i18n/locales/en.json index 4e110c67..1ebae892 100644 --- a/web/default/src/i18n/locales/en.json +++ b/web/default/src/i18n/locales/en.json @@ -5354,6 +5354,97 @@ "Zero retention": "Zero retention", "Zhipu": "Zhipu", "Zhipu V4": "Zhipu V4", - "Zoom": "Zoom" + "Zoom": "Zoom", + "Smart Operations": "Smart Operations", + "Smart Operations Center": "Smart Operations Center", + "Production performance": "Production performance", + "Production performance is not loaded automatically": "Production performance is not loaded automatically", + "Select Apply filters or Refresh to query the current time range.": "Select Apply filters or Refresh to query the current time range.", + "Read only": "Read only", + "Legacy log": "Legacy log", + "Log fallback mode": "Log fallback mode", + "This legacy view estimates outcomes from eligible Consume and Error logs. Known billing adjustments and channel probes are excluded, but it is not a complete Relay Attempt trace. Logged latency uses only non-retry samples and remains accurate only to whole seconds.": "This legacy view estimates outcomes from eligible Consume and Error logs. Known billing adjustments and channel probes are excluded, but it is not a complete Relay Attempt trace. Logged latency uses only non-retry samples and remains accurate only to whole seconds.", + "Retry and latency data are temporarily unavailable": "Retry and latency data are temporarily unavailable", + "Historical retry markers are still being prepared. Request counts and estimated success rates remain available; retry counts and logged latency will appear after the backfill completes.": "Historical retry markers are still being prepared. Request counts and estimated success rates remain available; retry counts and logged latency will appear after the backfill completes.", + "The retry marker readiness state could not be verified. Request counts and estimated success rates remain available, but retry counts and logged latency are unavailable; check the application and log databases.": "The retry marker readiness state could not be verified. Request counts and estimated success rates remain available, but retry counts and logged latency are unavailable; check the application and log databases.", + "Time range": "Time range", + "All channels": "All channels", + "All models": "All models", + "Apply filters": "Apply filters", + "Observed log events": "Observed log events", + "Consume logs": "Consume logs", + "Consumed quota": "Consumed quota", + "Estimated success rate": "Estimated success rate", + "Latency samples": "Latency samples", + "Last observed": "Last observed", + "Unable to load production performance": "Unable to load production performance", + "Check log database availability and administrator permissions, then try again.": "Check log database availability and administrator permissions, then try again.", + "No recorded production logs": "No recorded production logs", + "Try a wider time range or check whether consumption and error logging are enabled.": "Try a wider time range or check whether consumption and error logging are enabled.", + "{{count}} rows": "{{count}} rows", + "The table was truncated to {{count}} rows. Summary cards still cover all matching eligible logs.": "The table was truncated to {{count}} rows. Summary cards still cover all matching eligible logs.", + "Generated at {{time}}": "Generated at {{time}}", + "Production performance details": "Production performance details", + "Channel type": "Channel type", + "Error logs": "Error logs", + "Logged latency": "Logged latency", + "Retries": "Retries", + "Probe latency": "Probe latency", + "Data quality": "Data quality", + "Interpret with caution": "Interpret with caution", + "Probe latency is shown for comparison only and is not included in production log latency or success-rate calculations.": "Probe latency is shown for comparison only and is not included in production log latency or success-rate calculations.", + "Partial data": "Partial data", + "Weak correlation": "Weak correlation", + "Heuristic outcome": "Heuristic outcome", + "Coarse latency": "Coarse latency", + "Probe data is separate": "Probe data is separate", + "Known non-attempt logs excluded": "Known non-attempt logs excluded", + "Time range was limited": "Time range was limited", + "Results truncated": "Results truncated", + "Retry marker backfill pending": "Retry marker backfill pending", + "Retry metrics unavailable": "Retry metrics unavailable", + "Consumption logs disabled": "Consumption logs disabled", + "Error logs disabled": "Error logs disabled", + "Error log collection is disabled": "Error log collection is disabled", + "Errors may be incomplete": "Errors may be incomplete", + "Recorded error logs": "Recorded error logs", + "The displayed error count includes only recorded logs and may be incomplete. Estimated success rate is unavailable while error log collection is disabled.": "The displayed error count includes only recorded logs and may be incomplete. Estimated success rate is unavailable while error log collection is disabled.", + "Channel metadata missing": "Channel metadata missing", + "Probe unavailable": "Probe unavailable", + "Channel performance": "Channel performance", + "Channel performance details": "Channel performance details", + "Channel performance is not loaded automatically": "Channel performance is not loaded automatically", + "No recorded channel logs": "No recorded channel logs", + "Unable to load channel performance": "Unable to load channel performance", + "Model performance": "Model performance", + "Observed model log events": "Observed model log events", + "Unable to load model performance": "Unable to load model performance", + "Model performance is not loaded automatically": "Model performance is not loaded automatically", + "No recorded model logs": "No recorded model logs", + "Model performance details": "Model performance details", + "This legacy view aggregates estimated model outcomes from eligible Consume and Error logs. It is not a complete Relay Attempt trace, and logged latency remains accurate only to whole seconds.": "This legacy view aggregates estimated model outcomes from eligible Consume and Error logs. It is not a complete Relay Attempt trace, and logged latency remains accurate only to whole seconds.", + "Model results are aggregated from eligible logs and are not a complete Relay Attempt trace.": "Model results are aggregated from eligible logs and are not a complete Relay Attempt trace.", + "Inspect read-only production performance and operational evidence.": "Inspect read-only production performance and operational evidence.", + "Throughput unavailable": "Throughput unavailable", + "Model throughput could not be read from performance metrics. Other log-based results remain available; check performance metrics collection and database availability.": "Model throughput could not be read from performance metrics. Other log-based results remain available; check performance metrics collection and database availability.", + "Unable to load model performance details": "Unable to load model performance details", + "Check performance metrics collection and database availability, then try again.": "Check performance metrics collection and database availability, then try again.", + "Bucket-level estimate": "Bucket-level estimate", + "New performance samples are not being collected. Existing performance data may be incomplete.": "New performance samples are not being collected. Existing performance data may be incomplete.", + "No performance samples were recorded for this time range.": "No performance samples were recorded for this time range.", + "No throughput samples": "No throughput samples", + "Performance metrics collection is disabled": "Performance metrics collection is disabled", + "Performance metrics could not be queried. Log-based model results remain available; check the application database.": "Performance metrics could not be queried. Log-based model results remain available; check the application database.", + "Throughput coverage: {{start}} – {{end}} ({{minutes}} minute buckets).": "Throughput coverage: {{start}} – {{end}} ({{minutes}} minute buckets).", + "Throughput coverage: {{start}} – {{end}}. Bucket granularity is unknown or mixed, so this range is approximate.": "Throughput coverage: {{start}} – {{end}}. Bucket granularity is unknown or mixed, so this range is approximate.", + "Throughput query failed": "Throughput query failed", + "Last 24 hours": "Last 24 hours", + "Channel models and groups from eligible production logs": "Channel models and groups from eligible production logs", + "Unable to load channel performance details": "Unable to load channel performance details", + "No channel model performance": "No channel model performance", + "No eligible production logs were recorded for this channel in the last 24 hours.": "No eligible production logs were recorded for this channel in the last 24 hours.", + "The detail table was truncated to {{count}} rows. The summary still covers all matching eligible logs.": "The detail table was truncated to {{count}} rows. The summary still covers all matching eligible logs.", + "Throughput data is partial": "Throughput data is partial", + "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete.": "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete." } } diff --git a/web/default/src/i18n/locales/fr.json b/web/default/src/i18n/locales/fr.json index 24c7fc4d..b1db45ca 100644 --- a/web/default/src/i18n/locales/fr.json +++ b/web/default/src/i18n/locales/fr.json @@ -5354,6 +5354,97 @@ "Zero retention": "Aucune rétention", "Zhipu": "Zhipu", "Zhipu V4": "Zhipu V4", - "Zoom": "Zoom" + "Zoom": "Zoom", + "Smart Operations": "Opérations intelligentes", + "Smart Operations Center": "Centre des opérations intelligentes", + "Production performance": "Performances de production", + "Production performance is not loaded automatically": "Les performances de production ne sont pas chargées automatiquement", + "Select Apply filters or Refresh to query the current time range.": "Sélectionnez Appliquer les filtres ou Actualiser pour interroger la période actuelle.", + "Read only": "Lecture seule", + "Legacy log": "Journal hérité", + "Log fallback mode": "Mode de repli des journaux", + "This legacy view estimates outcomes from eligible Consume and Error logs. Known billing adjustments and channel probes are excluded, but it is not a complete Relay Attempt trace. Logged latency uses only non-retry samples and remains accurate only to whole seconds.": "Cette vue historique estime les résultats à partir des journaux Consume et Error éligibles. Les ajustements de facturation connus et les sondes de canal sont exclus, mais elle ne constitue pas une trace Relay Attempt complète. La latence journalisée utilise uniquement les échantillons sans nouvelle tentative et reste précise à la seconde entière seulement.", + "Retry and latency data are temporarily unavailable": "Les données de nouvelle tentative et de latence sont temporairement indisponibles", + "Historical retry markers are still being prepared. Request counts and estimated success rates remain available; retry counts and logged latency will appear after the backfill completes.": "Les marqueurs historiques de nouvelle tentative sont encore en cours de préparation. Le nombre de requêtes et les taux de réussite estimés restent disponibles ; le nombre de nouvelles tentatives et la latence journalisée apparaîtront une fois le remplissage terminé.", + "The retry marker readiness state could not be verified. Request counts and estimated success rates remain available, but retry counts and logged latency are unavailable; check the application and log databases.": "L'état de préparation des marqueurs de nouvelle tentative n'a pas pu être vérifié. Le nombre de requêtes et les taux de réussite estimés restent disponibles, mais le nombre de nouvelles tentatives et la latence journalisée sont indisponibles ; vérifiez les bases de données de l'application et des journaux.", + "Time range": "Période", + "All channels": "Tous les canaux", + "All models": "Tous les modèles", + "Apply filters": "Appliquer les filtres", + "Observed log events": "Événements de journal observés", + "Consume logs": "Journaux Consume", + "Consumed quota": "Quota consommé", + "Estimated success rate": "Taux de réussite estimé", + "Latency samples": "Échantillons de latence", + "Last observed": "Dernière observation", + "Unable to load production performance": "Impossible de charger les performances de production", + "Check log database availability and administrator permissions, then try again.": "Vérifiez la disponibilité de la base de journaux et les droits administrateur, puis réessayez.", + "No recorded production logs": "Aucun journal de production enregistré", + "Try a wider time range or check whether consumption and error logging are enabled.": "Élargissez la période ou vérifiez que les journaux de consommation et d'erreur sont activés.", + "{{count}} rows": "{{count}} lignes", + "The table was truncated to {{count}} rows. Summary cards still cover all matching eligible logs.": "Le tableau a été tronqué à {{count}} lignes. Les cartes récapitulatives couvrent toujours tous les journaux éligibles correspondants.", + "Generated at {{time}}": "Généré à {{time}}", + "Production performance details": "Détails des performances de production", + "Channel type": "Type de canal", + "Error logs": "Journaux d'erreur", + "Logged latency": "Latence journalisée", + "Retries": "Tentatives", + "Probe latency": "Latence de sonde", + "Data quality": "Qualité des données", + "Interpret with caution": "À interpréter avec prudence", + "Probe latency is shown for comparison only and is not included in production log latency or success-rate calculations.": "La latence de sonde est fournie à titre comparatif et n'entre pas dans les calculs de latence ou de taux de succès de production.", + "Partial data": "Données partielles", + "Weak correlation": "Corrélation faible", + "Heuristic outcome": "Résultat heuristique", + "Coarse latency": "Latence approximative", + "Probe data is separate": "Données de sonde séparées", + "Known non-attempt logs excluded": "Journaux connus hors tentative exclus", + "Time range was limited": "Période limitée", + "Results truncated": "Résultats tronqués", + "Retry marker backfill pending": "Remplissage des marqueurs de nouvelle tentative en cours", + "Retry metrics unavailable": "Métriques de nouvelle tentative indisponibles", + "Consumption logs disabled": "Journaux de consommation désactivés", + "Error logs disabled": "Journaux d'erreur désactivés", + "Error log collection is disabled": "La collecte des journaux d'erreur est désactivée", + "Errors may be incomplete": "Les erreurs peuvent être incomplètes", + "Recorded error logs": "Journaux d'erreur enregistrés", + "The displayed error count includes only recorded logs and may be incomplete. Estimated success rate is unavailable while error log collection is disabled.": "Le nombre d'erreurs affiché comprend uniquement les journaux enregistrés et peut être incomplet. Le taux de réussite estimé est indisponible tant que la collecte des journaux d'erreur est désactivée.", + "Channel metadata missing": "Métadonnées du canal manquantes", + "Probe unavailable": "Sonde indisponible", + "Channel performance": "Performance des canaux", + "Channel performance details": "Détails des performances des canaux", + "Channel performance is not loaded automatically": "Les performances des canaux ne sont pas chargées automatiquement", + "No recorded channel logs": "Aucun journal de canal enregistré", + "Unable to load channel performance": "Impossible de charger les performances des canaux", + "Model performance": "Performance des modèles", + "Observed model log events": "Événements de journal des modèles observés", + "Unable to load model performance": "Impossible de charger les performances des modèles", + "Model performance is not loaded automatically": "Les performances des modèles ne sont pas chargées automatiquement", + "No recorded model logs": "Aucun journal de modèle enregistré", + "Model performance details": "Détails des performances du modèle", + "This legacy view aggregates estimated model outcomes from eligible Consume and Error logs. It is not a complete Relay Attempt trace, and logged latency remains accurate only to whole seconds.": "Cette vue héritée agrège les résultats estimés des modèles à partir des journaux Consume et Error admissibles. Elle ne constitue pas une trace Relay Attempt complète et la latence journalisée n'est précise qu'à la seconde.", + "Model results are aggregated from eligible logs and are not a complete Relay Attempt trace.": "Les résultats des modèles sont agrégés à partir des journaux admissibles et ne constituent pas une trace Relay Attempt complète.", + "Inspect read-only production performance and operational evidence.": "Examiner les performances de production et les preuves opérationnelles en lecture seule.", + "Throughput unavailable": "Débit indisponible", + "Model throughput could not be read from performance metrics. Other log-based results remain available; check performance metrics collection and database availability.": "Le débit des modèles n'a pas pu être lu depuis les métriques de performance. Les autres résultats fondés sur les journaux restent disponibles ; vérifiez la collecte des métriques et la disponibilité de la base de données.", + "Unable to load model performance details": "Impossible de charger les détails des performances du modèle", + "Check performance metrics collection and database availability, then try again.": "Vérifiez la collecte des métriques de performance et la disponibilité de la base de données, puis réessayez.", + "Bucket-level estimate": "Estimation par compartiments", + "New performance samples are not being collected. Existing performance data may be incomplete.": "Aucun nouvel échantillon de performance n'est collecté. Les données existantes peuvent être incomplètes.", + "No performance samples were recorded for this time range.": "Aucun échantillon de performance n'a été enregistré pour cette période.", + "No throughput samples": "Aucun échantillon de débit", + "Performance metrics collection is disabled": "La collecte des métriques de performance est désactivée", + "Performance metrics could not be queried. Log-based model results remain available; check the application database.": "Les métriques de performance n'ont pas pu être interrogées. Les résultats fondés sur les journaux restent disponibles ; vérifiez la base de données de l'application.", + "Throughput coverage: {{start}} – {{end}} ({{minutes}} minute buckets).": "Couverture du débit : {{start}} – {{end}} (compartiments de {{minutes}} minutes).", + "Throughput coverage: {{start}} – {{end}}. Bucket granularity is unknown or mixed, so this range is approximate.": "Couverture du débit : {{start}} – {{end}}. La granularité des compartiments est inconnue ou mixte ; cette plage est donc approximative.", + "Throughput query failed": "Échec de la requête de débit", + "Last 24 hours": "Dernières 24 heures", + "Channel models and groups from eligible production logs": "Modèles et groupes du canal issus des journaux de production éligibles", + "Unable to load channel performance details": "Impossible de charger les détails des performances du canal", + "No channel model performance": "Aucune performance de modèle de canal", + "No eligible production logs were recorded for this channel in the last 24 hours.": "Aucun journal de production éligible n'a été enregistré pour ce canal au cours des dernières 24 heures.", + "The detail table was truncated to {{count}} rows. The summary still covers all matching eligible logs.": "Le tableau détaillé a été limité à {{count}} lignes. Le résumé couvre toujours tous les journaux éligibles correspondants.", + "Throughput data is partial": "Données de débit partielles", + "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete.": "Les échantillons stockés et locaux sont affichés, mais les compartiments actifs partagés sont illisibles. Le débit multi-nœuds récent peut être incomplet." } } diff --git a/web/default/src/i18n/locales/ja.json b/web/default/src/i18n/locales/ja.json index c212ba80..c49c9d1b 100644 --- a/web/default/src/i18n/locales/ja.json +++ b/web/default/src/i18n/locales/ja.json @@ -5354,6 +5354,97 @@ "Zero retention": "データ保持なし", "Zhipu": "Zhipu", "Zhipu V4": "Zhipu V 4", - "Zoom": "ズーム" + "Zoom": "ズーム", + "Smart Operations": "スマート運用", + "Smart Operations Center": "スマート運用センター", + "Production performance": "本番性能", + "Production performance is not loaded automatically": "本番性能は自動では読み込まれません", + "Select Apply filters or Refresh to query the current time range.": "「フィルターを適用」または「更新」を選択して、現在の期間を照会してください。", + "Read only": "読み取り専用", + "Legacy log": "レガシーログ", + "Log fallback mode": "ログフォールバックモード", + "This legacy view estimates outcomes from eligible Consume and Error logs. Known billing adjustments and channel probes are excluded, but it is not a complete Relay Attempt trace. Logged latency uses only non-retry samples and remains accurate only to whole seconds.": "このレガシービューは、対象となる Consume および Error ログから結果を推定します。既知の課金調整とチャネルプローブは除外していますが、完全な Relay Attempt トレースではありません。ログ遅延はリトライのないサンプルのみを使用し、精度は引き続き整数秒単位です。", + "Retry and latency data are temporarily unavailable": "リトライと遅延のデータは一時的に利用できません", + "Historical retry markers are still being prepared. Request counts and estimated success rates remain available; retry counts and logged latency will appear after the backfill completes.": "過去のリトライマーカーを準備しています。リクエスト数と推定成功率は引き続き利用でき、バックフィル完了後にリトライ回数とログ遅延が表示されます。", + "The retry marker readiness state could not be verified. Request counts and estimated success rates remain available, but retry counts and logged latency are unavailable; check the application and log databases.": "リトライマーカーの準備状態を確認できませんでした。リクエスト数と推定成功率は利用できますが、リトライ回数とログ遅延は利用できません。アプリケーションDBとログDBを確認してください。", + "Time range": "期間", + "All channels": "すべてのチャネル", + "All models": "すべてのモデル", + "Apply filters": "フィルターを適用", + "Observed log events": "観測ログイベント", + "Consume logs": "Consumeログ", + "Consumed quota": "消費クォータ", + "Estimated success rate": "推定成功率", + "Latency samples": "レイテンシーサンプル", + "Last observed": "最終観測", + "Unable to load production performance": "本番性能を読み込めません", + "Check log database availability and administrator permissions, then try again.": "ログデータベースの可用性と管理者権限を確認して、もう一度お試しください。", + "No recorded production logs": "記録済みの本番ログはありません", + "Try a wider time range or check whether consumption and error logging are enabled.": "期間を広げるか、消費ログとエラーログが有効か確認してください。", + "{{count}} rows": "{{count}} 行", + "The table was truncated to {{count}} rows. Summary cards still cover all matching eligible logs.": "テーブルは{{count}}行に切り詰められました。サマリーカードは一致する対象ログ全体を引き続き集計しています。", + "Generated at {{time}}": "生成日時：{{time}}", + "Production performance details": "本番性能の詳細", + "Channel type": "チャネル種別", + "Error logs": "エラーログ", + "Logged latency": "ログ遅延", + "Retries": "リトライ", + "Probe latency": "プローブ遅延", + "Data quality": "データ品質", + "Interpret with caution": "注意して解釈してください", + "Probe latency is shown for comparison only and is not included in production log latency or success-rate calculations.": "プローブ遅延は比較用であり、本番ログの遅延や成功率の計算には含まれません。", + "Partial data": "部分データ", + "Weak correlation": "相関が弱い", + "Heuristic outcome": "推定結果", + "Coarse latency": "粗い遅延", + "Probe data is separate": "プローブデータは別表示", + "Known non-attempt logs excluded": "既知の非試行ログを除外", + "Time range was limited": "期間が制限されています", + "Results truncated": "結果を切り詰め", + "Retry marker backfill pending": "リトライマーカーをバックフィル中", + "Retry metrics unavailable": "リトライ指標を利用不可", + "Consumption logs disabled": "消費ログは無効です", + "Error logs disabled": "エラーログは無効です", + "Error log collection is disabled": "エラーログ収集が無効です", + "Errors may be incomplete": "エラーが不完全な可能性があります", + "Recorded error logs": "記録済みエラーログ", + "The displayed error count includes only recorded logs and may be incomplete. Estimated success rate is unavailable while error log collection is disabled.": "表示されるエラー数は記録済みログのみを含むため、不完全な可能性があります。エラーログ収集が無効な間は推定成功率を利用できません。", + "Channel metadata missing": "チャネルメタデータがありません", + "Probe unavailable": "プローブデータなし", + "Channel performance": "チャネル性能", + "Channel performance details": "チャネル性能の詳細", + "Channel performance is not loaded automatically": "チャネル性能は自動的に読み込まれません", + "No recorded channel logs": "記録されたチャネルログはありません", + "Unable to load channel performance": "チャネル性能を読み込めません", + "Model performance": "モデル性能", + "Observed model log events": "観測モデルログイベント", + "Unable to load model performance": "モデル性能を読み込めません", + "Model performance is not loaded automatically": "モデル性能は自動的に読み込まれません", + "No recorded model logs": "記録済みのモデルログはありません", + "Model performance details": "モデル性能の詳細", + "This legacy view aggregates estimated model outcomes from eligible Consume and Error logs. It is not a complete Relay Attempt trace, and logged latency remains accurate only to whole seconds.": "このレガシービューは、対象となる Consume および Error ログからモデル結果を推定して集計します。完全な Relay Attempt トレースではなく、記録されたレイテンシーの精度は秒単位です。", + "Model results are aggregated from eligible logs and are not a complete Relay Attempt trace.": "モデル結果は対象ログから集計されたもので、完全な Relay Attempt トレースではありません。", + "Inspect read-only production performance and operational evidence.": "読み取り専用の本番性能と運用証拠を確認します。", + "Throughput unavailable": "スループットを利用できません", + "Model throughput could not be read from performance metrics. Other log-based results remain available; check performance metrics collection and database availability.": "パフォーマンスメトリクスからモデルのスループットを読み取れませんでした。ログベースの他の結果は利用できます。パフォーマンスメトリクスの収集とデータベースの可用性を確認してください。", + "Unable to load model performance details": "モデル性能の詳細を読み込めません", + "Check performance metrics collection and database availability, then try again.": "パフォーマンスメトリクスの収集とデータベースの可用性を確認してから、もう一度お試しください。", + "Bucket-level estimate": "時間バケット単位の推定", + "New performance samples are not being collected. Existing performance data may be incomplete.": "新しいパフォーマンスサンプルは収集されていません。既存のデータは不完全な可能性があります。", + "No performance samples were recorded for this time range.": "この期間にはパフォーマンスサンプルが記録されていません。", + "No throughput samples": "スループットサンプルなし", + "Performance metrics collection is disabled": "パフォーマンスメトリクスの収集は無効です", + "Performance metrics could not be queried. Log-based model results remain available; check the application database.": "パフォーマンスメトリクスを照会できませんでした。ログベースのモデル結果は利用できます。アプリケーションデータベースを確認してください。", + "Throughput coverage: {{start}} – {{end}} ({{minutes}} minute buckets).": "スループットの対象範囲：{{start}} – {{end}}（{{minutes}}分バケット）。", + "Throughput coverage: {{start}} – {{end}}. Bucket granularity is unknown or mixed, so this range is approximate.": "スループットの対象範囲：{{start}} – {{end}}。バケット粒度が不明または混在しているため、この範囲は概算です。", + "Throughput query failed": "スループットの照会に失敗しました", + "Last 24 hours": "過去24時間", + "Channel models and groups from eligible production logs": "対象となる本番ログのチャネルモデルとグループ", + "Unable to load channel performance details": "チャネル性能の詳細を読み込めません", + "No channel model performance": "チャネルモデルの性能データはありません", + "No eligible production logs were recorded for this channel in the last 24 hours.": "過去24時間にこのチャネルの対象本番ログは記録されていません。", + "The detail table was truncated to {{count}} rows. The summary still covers all matching eligible logs.": "詳細テーブルは{{count}}行に制限されています。概要は一致する対象ログをすべてカバーします。", + "Throughput data is partial": "スループットデータが不完全です", + "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete.": "保存済みおよびローカルの性能サンプルを表示していますが、共有アクティブバケットを読み取れませんでした。最近のマルチノードスループットは不完全な可能性があります。" } } diff --git a/web/default/src/i18n/locales/ru.json b/web/default/src/i18n/locales/ru.json index aafa73d8..97b92d7d 100644 --- a/web/default/src/i18n/locales/ru.json +++ b/web/default/src/i18n/locales/ru.json @@ -5354,6 +5354,97 @@ "Zero retention": "Без хранения данных", "Zhipu": "Zhipu", "Zhipu V4": "Zhipu V4", - "Zoom": "Zoom" + "Zoom": "Zoom", + "Smart Operations": "Интеллектуальные операции", + "Smart Operations Center": "Центр интеллектуальных операций", + "Production performance": "Производительность в продакшене", + "Production performance is not loaded automatically": "Данные о производительности не загружаются автоматически", + "Select Apply filters or Refresh to query the current time range.": "Нажмите «Применить фильтры» или «Обновить», чтобы запросить данные за текущий период.", + "Read only": "Только чтение", + "Legacy log": "Устаревший журнал", + "Log fallback mode": "Режим резервных журналов", + "This legacy view estimates outcomes from eligible Consume and Error logs. Known billing adjustments and channel probes are excluded, but it is not a complete Relay Attempt trace. Logged latency uses only non-retry samples and remains accurate only to whole seconds.": "Это представление на основе устаревших журналов оценивает результаты по подходящим записям Consume и Error. Известные корректировки биллинга и проверки каналов исключены, но это не полная трассировка Relay Attempt. Задержка по журналам использует только образцы без повторов, а ее точность по-прежнему ограничена целыми секундами.", + "Retry and latency data are temporarily unavailable": "Данные о повторах и задержке временно недоступны", + "Historical retry markers are still being prepared. Request counts and estimated success rates remain available; retry counts and logged latency will appear after the backfill completes.": "Исторические маркеры повторов еще подготавливаются. Количество запросов и расчетный показатель успешности остаются доступными; количество повторов и задержка по журналам появятся после завершения обратного заполнения.", + "The retry marker readiness state could not be verified. Request counts and estimated success rates remain available, but retry counts and logged latency are unavailable; check the application and log databases.": "Не удалось проверить готовность маркеров повторов. Количество запросов и расчетный показатель успешности остаются доступными, но количество повторов и задержка по журналам недоступны; проверьте базу приложения и базу журналов.", + "Time range": "Период", + "All channels": "Все каналы", + "All models": "Все модели", + "Apply filters": "Применить фильтры", + "Observed log events": "Наблюдаемые события журнала", + "Consume logs": "Записи Consume", + "Consumed quota": "Использованная квота", + "Estimated success rate": "Расчетная доля успеха", + "Latency samples": "Образцы задержки", + "Last observed": "Последнее наблюдение", + "Unable to load production performance": "Не удалось загрузить производительность в продакшене", + "Check log database availability and administrator permissions, then try again.": "Проверьте доступность базы журналов и права администратора, затем повторите попытку.", + "No recorded production logs": "Нет записанных производственных журналов", + "Try a wider time range or check whether consumption and error logging are enabled.": "Расширьте период или проверьте, включены ли журналы потребления и ошибок.", + "{{count}} rows": "Строк: {{count}}", + "The table was truncated to {{count}} rows. Summary cards still cover all matching eligible logs.": "Таблица усечена до {{count}} строк. Сводные карточки по-прежнему охватывают все подходящие совпавшие записи.", + "Generated at {{time}}": "Создано: {{time}}", + "Production performance details": "Подробности производительности", + "Channel type": "Тип канала", + "Error logs": "Журналы ошибок", + "Logged latency": "Задержка по журналу", + "Retries": "Повторы", + "Probe latency": "Задержка проверки", + "Data quality": "Качество данных", + "Interpret with caution": "Интерпретируйте с осторожностью", + "Probe latency is shown for comparison only and is not included in production log latency or success-rate calculations.": "Задержка проверки показана только для сравнения и не входит в расчёты производственной задержки или доли успеха.", + "Partial data": "Неполные данные", + "Weak correlation": "Слабая корреляция", + "Heuristic outcome": "Эвристический результат", + "Coarse latency": "Грубая задержка", + "Probe data is separate": "Данные проверки отдельно", + "Known non-attempt logs excluded": "Известные записи, не являющиеся попытками, исключены", + "Time range was limited": "Период ограничен", + "Results truncated": "Результаты усечены", + "Retry marker backfill pending": "Идет обратное заполнение маркеров повторов", + "Retry metrics unavailable": "Метрики повторов недоступны", + "Consumption logs disabled": "Журналы потребления отключены", + "Error logs disabled": "Журналы ошибок отключены", + "Error log collection is disabled": "Сбор журналов ошибок отключен", + "Errors may be incomplete": "Ошибки могут быть неполными", + "Recorded error logs": "Записанные журналы ошибок", + "The displayed error count includes only recorded logs and may be incomplete. Estimated success rate is unavailable while error log collection is disabled.": "Показанное число ошибок включает только записанные журналы и может быть неполным. Расчетный показатель успешности недоступен, пока сбор журналов ошибок отключен.", + "Channel metadata missing": "Нет метаданных канала", + "Probe unavailable": "Данные проверки недоступны", + "Channel performance": "Производительность каналов", + "Channel performance details": "Сведения о производительности каналов", + "Channel performance is not loaded automatically": "Производительность каналов не загружается автоматически", + "No recorded channel logs": "Нет записанных журналов каналов", + "Unable to load channel performance": "Не удалось загрузить производительность каналов", + "Model performance": "Производительность моделей", + "Observed model log events": "Наблюдаемые события журналов моделей", + "Unable to load model performance": "Не удалось загрузить производительность моделей", + "Model performance is not loaded automatically": "Производительность моделей не загружается автоматически", + "No recorded model logs": "Нет записанных журналов моделей", + "Model performance details": "Сведения о производительности модели", + "This legacy view aggregates estimated model outcomes from eligible Consume and Error logs. It is not a complete Relay Attempt trace, and logged latency remains accurate only to whole seconds.": "Это устаревшее представление агрегирует расчетные результаты моделей по подходящим журналам Consume и Error. Оно не является полной трассировкой Relay Attempt, а задержка в журнале точна только до целых секунд.", + "Model results are aggregated from eligible logs and are not a complete Relay Attempt trace.": "Результаты моделей агрегируются по подходящим журналам и не являются полной трассировкой Relay Attempt.", + "Inspect read-only production performance and operational evidence.": "Просматривайте производительность и операционные свидетельства в режиме только чтения.", + "Throughput unavailable": "Пропускная способность недоступна", + "Model throughput could not be read from performance metrics. Other log-based results remain available; check performance metrics collection and database availability.": "Не удалось прочитать пропускную способность моделей из метрик производительности. Остальные результаты на основе журналов доступны; проверьте сбор метрик производительности и доступность базы данных.", + "Unable to load model performance details": "Не удалось загрузить сведения о производительности модели", + "Check performance metrics collection and database availability, then try again.": "Проверьте сбор метрик производительности и доступность базы данных, затем повторите попытку.", + "Bucket-level estimate": "Оценка по временным блокам", + "New performance samples are not being collected. Existing performance data may be incomplete.": "Новые образцы производительности не собираются. Существующие данные могут быть неполными.", + "No performance samples were recorded for this time range.": "За этот период образцы производительности не зарегистрированы.", + "No throughput samples": "Нет образцов пропускной способности", + "Performance metrics collection is disabled": "Сбор метрик производительности отключен", + "Performance metrics could not be queried. Log-based model results remain available; check the application database.": "Не удалось запросить метрики производительности. Результаты на основе журналов доступны; проверьте базу данных приложения.", + "Throughput coverage: {{start}} – {{end}} ({{minutes}} minute buckets).": "Охват пропускной способности: {{start}} – {{end}} (блоки по {{minutes}} мин.).", + "Throughput coverage: {{start}} – {{end}}. Bucket granularity is unknown or mixed, so this range is approximate.": "Охват пропускной способности: {{start}} – {{end}}. Гранулярность интервалов неизвестна или смешана, поэтому диапазон является приблизительным.", + "Throughput query failed": "Сбой запроса пропускной способности", + "Last 24 hours": "Последние 24 часа", + "Channel models and groups from eligible production logs": "Модели и группы канала по подходящим производственным журналам", + "Unable to load channel performance details": "Не удалось загрузить подробности производительности канала", + "No channel model performance": "Нет данных о производительности моделей канала", + "No eligible production logs were recorded for this channel in the last 24 hours.": "За последние 24 часа для этого канала не записано подходящих производственных журналов.", + "The detail table was truncated to {{count}} rows. The summary still covers all matching eligible logs.": "Таблица подробностей ограничена {{count}} строками. Сводка по-прежнему охватывает все подходящие журналы.", + "Throughput data is partial": "Данные о пропускной способности неполны", + "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete.": "Показаны сохранённые и локальные образцы производительности, но общие активные сегменты не удалось прочитать. Последняя многонодовая пропускная способность может быть неполной." } } diff --git a/web/default/src/i18n/locales/vi.json b/web/default/src/i18n/locales/vi.json index 7c4f0cb6..0f420716 100644 --- a/web/default/src/i18n/locales/vi.json +++ b/web/default/src/i18n/locales/vi.json @@ -5354,6 +5354,97 @@ "Zero retention": "Không lưu dữ liệu", "Zhipu": "Zhipu", "Zhipu V4": "Zhipu V4", - "Zoom": "Zoom" + "Zoom": "Zoom", + "Smart Operations": "Vận hành thông minh", + "Smart Operations Center": "Trung tâm vận hành thông minh", + "Production performance": "Hiệu năng thực tế", + "Production performance is not loaded automatically": "Dữ liệu hiệu năng thực tế không được tải tự động", + "Select Apply filters or Refresh to query the current time range.": "Chọn Áp dụng bộ lọc hoặc Làm mới để truy vấn khoảng thời gian hiện tại.", + "Read only": "Chỉ đọc", + "Legacy log": "Nhật ký cũ", + "Log fallback mode": "Chế độ dự phòng bằng nhật ký", + "This legacy view estimates outcomes from eligible Consume and Error logs. Known billing adjustments and channel probes are excluded, but it is not a complete Relay Attempt trace. Logged latency uses only non-retry samples and remains accurate only to whole seconds.": "Chế độ xem legacy này ước tính kết quả từ các nhật ký Consume và Error đủ điều kiện. Các điều chỉnh thanh toán đã biết và lượt kiểm tra kênh được loại trừ, nhưng đây không phải là dấu vết Relay Attempt đầy đủ. Độ trễ ghi nhật ký chỉ sử dụng các mẫu không thử lại và vẫn có độ chính xác đến giây nguyên.", + "Retry and latency data are temporarily unavailable": "Dữ liệu thử lại và độ trễ tạm thời không khả dụng", + "Historical retry markers are still being prepared. Request counts and estimated success rates remain available; retry counts and logged latency will appear after the backfill completes.": "Các dấu thử lại lịch sử vẫn đang được chuẩn bị. Số lượng yêu cầu và tỷ lệ thành công ước tính vẫn khả dụng; số lần thử lại và độ trễ ghi nhận sẽ xuất hiện sau khi hoàn tất điền bù.", + "The retry marker readiness state could not be verified. Request counts and estimated success rates remain available, but retry counts and logged latency are unavailable; check the application and log databases.": "Không thể xác minh trạng thái sẵn sàng của dấu thử lại. Số lượng yêu cầu và tỷ lệ thành công ước tính vẫn khả dụng, nhưng số lần thử lại và độ trễ ghi nhận chưa khả dụng; hãy kiểm tra cơ sở dữ liệu ứng dụng và nhật ký.", + "Time range": "Khoảng thời gian", + "All channels": "Tất cả kênh", + "All models": "Tất cả mô hình", + "Apply filters": "Áp dụng bộ lọc", + "Observed log events": "Sự kiện nhật ký quan sát được", + "Consume logs": "Nhật ký Consume", + "Consumed quota": "Hạn mức đã tiêu thụ", + "Estimated success rate": "Tỷ lệ thành công ước tính", + "Latency samples": "Mẫu độ trễ", + "Last observed": "Quan sát gần nhất", + "Unable to load production performance": "Không thể tải hiệu năng thực tế", + "Check log database availability and administrator permissions, then try again.": "Hãy kiểm tra cơ sở dữ liệu nhật ký và quyền quản trị rồi thử lại.", + "No recorded production logs": "Không có nhật ký thực tế đã ghi", + "Try a wider time range or check whether consumption and error logging are enabled.": "Hãy mở rộng khoảng thời gian hoặc kiểm tra nhật ký tiêu thụ và lỗi đã được bật chưa.", + "{{count}} rows": "{{count}} hàng", + "The table was truncated to {{count}} rows. Summary cards still cover all matching eligible logs.": "Bảng đã được cắt còn {{count}} hàng. Các thẻ tóm tắt vẫn bao phủ toàn bộ nhật ký đủ điều kiện khớp bộ lọc.", + "Generated at {{time}}": "Tạo lúc {{time}}", + "Production performance details": "Chi tiết hiệu năng thực tế", + "Channel type": "Loại kênh", + "Error logs": "Nhật ký lỗi", + "Logged latency": "Độ trễ theo nhật ký", + "Retries": "Số lần thử lại", + "Probe latency": "Độ trễ kiểm tra", + "Data quality": "Chất lượng dữ liệu", + "Interpret with caution": "Cần diễn giải thận trọng", + "Probe latency is shown for comparison only and is not included in production log latency or success-rate calculations.": "Độ trễ kiểm tra chỉ dùng để so sánh và không được tính vào độ trễ hoặc tỷ lệ thành công thực tế.", + "Partial data": "Dữ liệu một phần", + "Weak correlation": "Tương quan yếu", + "Heuristic outcome": "Kết quả suy đoán", + "Coarse latency": "Độ trễ thô", + "Probe data is separate": "Dữ liệu kiểm tra tách biệt", + "Known non-attempt logs excluded": "Đã loại trừ nhật ký không phải lần thử đã biết", + "Time range was limited": "Khoảng thời gian đã bị giới hạn", + "Results truncated": "Kết quả bị cắt bớt", + "Retry marker backfill pending": "Đang điền bù dấu thử lại", + "Retry metrics unavailable": "Chỉ số thử lại không khả dụng", + "Consumption logs disabled": "Nhật ký tiêu thụ đã tắt", + "Error logs disabled": "Nhật ký lỗi đã tắt", + "Error log collection is disabled": "Tính năng thu thập nhật ký lỗi đang tắt", + "Errors may be incomplete": "Dữ liệu lỗi có thể chưa đầy đủ", + "Recorded error logs": "Nhật ký lỗi đã ghi nhận", + "The displayed error count includes only recorded logs and may be incomplete. Estimated success rate is unavailable while error log collection is disabled.": "Số lỗi hiển thị chỉ bao gồm các nhật ký đã ghi nhận và có thể chưa đầy đủ. Tỷ lệ thành công ước tính không khả dụng khi tính năng thu thập nhật ký lỗi đang tắt.", + "Channel metadata missing": "Thiếu siêu dữ liệu kênh", + "Probe unavailable": "Không có dữ liệu kiểm tra", + "Channel performance": "Hiệu năng kênh", + "Channel performance details": "Chi tiết hiệu năng kênh", + "Channel performance is not loaded automatically": "Hiệu năng kênh không được tải tự động", + "No recorded channel logs": "Không có nhật ký kênh đã ghi", + "Unable to load channel performance": "Không thể tải hiệu năng kênh", + "Model performance": "Hiệu năng mô hình", + "Observed model log events": "Sự kiện nhật ký mô hình được quan sát", + "Unable to load model performance": "Không thể tải hiệu năng mô hình", + "Model performance is not loaded automatically": "Hiệu năng mô hình không tự động tải", + "No recorded model logs": "Không có nhật ký mô hình đã ghi", + "Model performance details": "Chi tiết hiệu năng mô hình", + "This legacy view aggregates estimated model outcomes from eligible Consume and Error logs. It is not a complete Relay Attempt trace, and logged latency remains accurate only to whole seconds.": "Chế độ xem cũ này tổng hợp kết quả mô hình ước tính từ các nhật ký Consume và Error hợp lệ. Đây không phải là dấu vết Relay Attempt đầy đủ và độ trễ trong nhật ký chỉ chính xác đến giây.", + "Model results are aggregated from eligible logs and are not a complete Relay Attempt trace.": "Kết quả mô hình được tổng hợp từ các nhật ký hợp lệ và không phải là dấu vết Relay Attempt đầy đủ.", + "Inspect read-only production performance and operational evidence.": "Xem hiệu năng thực tế và bằng chứng vận hành ở chế độ chỉ đọc.", + "Throughput unavailable": "Không có thông lượng", + "Model throughput could not be read from performance metrics. Other log-based results remain available; check performance metrics collection and database availability.": "Không thể đọc thông lượng mô hình từ các chỉ số hiệu năng. Các kết quả dựa trên nhật ký khác vẫn khả dụng; hãy kiểm tra việc thu thập chỉ số hiệu năng và khả dụng của cơ sở dữ liệu.", + "Unable to load model performance details": "Không thể tải chi tiết hiệu năng mô hình", + "Check performance metrics collection and database availability, then try again.": "Hãy kiểm tra việc thu thập chỉ số hiệu năng và khả dụng của cơ sở dữ liệu, sau đó thử lại.", + "Bucket-level estimate": "Ước tính theo nhóm thời gian", + "New performance samples are not being collected. Existing performance data may be incomplete.": "Hệ thống không thu thập mẫu hiệu năng mới. Dữ liệu hiện có có thể chưa đầy đủ.", + "No performance samples were recorded for this time range.": "Không có mẫu hiệu năng nào được ghi nhận trong khoảng thời gian này.", + "No throughput samples": "Không có mẫu thông lượng", + "Performance metrics collection is disabled": "Đã tắt thu thập chỉ số hiệu năng", + "Performance metrics could not be queried. Log-based model results remain available; check the application database.": "Không thể truy vấn chỉ số hiệu năng. Kết quả mô hình dựa trên nhật ký vẫn khả dụng; hãy kiểm tra cơ sở dữ liệu ứng dụng.", + "Throughput coverage: {{start}} – {{end}} ({{minutes}} minute buckets).": "Phạm vi thông lượng: {{start}} – {{end}} (nhóm {{minutes}} phút).", + "Throughput coverage: {{start}} – {{end}}. Bucket granularity is unknown or mixed, so this range is approximate.": "Phạm vi thông lượng: {{start}} – {{end}}. Độ chi tiết của nhóm dữ liệu không xác định hoặc không đồng nhất, vì vậy phạm vi này chỉ là ước tính.", + "Throughput query failed": "Truy vấn thông lượng thất bại", + "Last 24 hours": "24 giờ qua", + "Channel models and groups from eligible production logs": "Mô hình và nhóm kênh từ nhật ký sản xuất hợp lệ", + "Unable to load channel performance details": "Không thể tải chi tiết hiệu suất kênh", + "No channel model performance": "Không có dữ liệu hiệu suất mô hình kênh", + "No eligible production logs were recorded for this channel in the last 24 hours.": "Không có nhật ký sản xuất hợp lệ nào được ghi cho kênh này trong 24 giờ qua.", + "The detail table was truncated to {{count}} rows. The summary still covers all matching eligible logs.": "Bảng chi tiết bị giới hạn ở {{count}} dòng. Phần tóm tắt vẫn bao phủ tất cả nhật ký hợp lệ phù hợp.", + "Throughput data is partial": "Dữ liệu thông lượng không đầy đủ", + "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete.": "Các mẫu hiệu suất đã lưu và cục bộ được hiển thị, nhưng không thể đọc các bucket hoạt động dùng chung. Thông lượng đa nút gần đây có thể không đầy đủ." } } diff --git a/web/default/src/i18n/locales/zh.json b/web/default/src/i18n/locales/zh.json index 8e623083..4461f65d 100644 --- a/web/default/src/i18n/locales/zh.json +++ b/web/default/src/i18n/locales/zh.json @@ -5354,6 +5354,97 @@ "Zero retention": "零数据保留", "Zhipu": "智谱", "Zhipu V4": "智谱 V4", - "Zoom": "缩放" + "Zoom": "缩放", + "Smart Operations": "智能运维", + "Smart Operations Center": "智能运维中心", + "Production performance": "生产性能", + "Production performance is not loaded automatically": "生产性能不会自动加载", + "Select Apply filters or Refresh to query the current time range.": "请选择“应用筛选”或“刷新”，查询当前时间范围的数据。", + "Read only": "只读", + "Legacy log": "日志回退", + "Log fallback mode": "日志回退模式", + "This legacy view estimates outcomes from eligible Consume and Error logs. Known billing adjustments and channel probes are excluded, but it is not a complete Relay Attempt trace. Logged latency uses only non-retry samples and remains accurate only to whole seconds.": "此日志回退视图根据符合条件的 Consume 与 Error 日志估算调用结果。已排除明确的账务调整和渠道探测，但它并非完整的 Relay Attempt 链路；日志延迟仅使用非重试样本，且精度仍仅到整秒。", + "Retry and latency data are temporarily unavailable": "重试和延迟数据暂不可用", + "Historical retry markers are still being prepared. Request counts and estimated success rates remain available; retry counts and logged latency will appear after the backfill completes.": "历史重试标记仍在准备中。请求计数和估算成功率仍可用；回填完成后将显示重试次数和日志延迟。", + "The retry marker readiness state could not be verified. Request counts and estimated success rates remain available, but retry counts and logged latency are unavailable; check the application and log databases.": "无法确认重试标记的就绪状态。请求计数和估算成功率仍可用，但重试次数和日志延迟暂不可用；请检查应用数据库和日志数据库。", + "Time range": "时间范围", + "All channels": "全部渠道", + "All models": "全部模型", + "Apply filters": "应用筛选", + "Observed log events": "观测日志事件", + "Consume logs": "消费日志", + "Consumed quota": "消耗额度", + "Estimated success rate": "估算成功率", + "Latency samples": "延迟样本", + "Last observed": "最近观测", + "Unable to load production performance": "无法加载生产性能", + "Check log database availability and administrator permissions, then try again.": "请检查日志数据库可用性和管理员权限后重试。", + "No recorded production logs": "没有已记录的生产日志", + "Try a wider time range or check whether consumption and error logging are enabled.": "请扩大时间范围，或检查消费日志与错误日志是否启用。", + "{{count}} rows": "{{count}} 行", + "The table was truncated to {{count}} rows. Summary cards still cover all matching eligible logs.": "表格已截断为 {{count}} 行，但摘要卡片仍覆盖所有符合条件的匹配日志。", + "Generated at {{time}}": "生成时间：{{time}}", + "Production performance details": "生产性能详情", + "Channel type": "渠道类型", + "Error logs": "错误日志", + "Logged latency": "日志延迟", + "Retries": "重试次数", + "Probe latency": "探测延迟", + "Data quality": "数据质量", + "Interpret with caution": "请谨慎解读", + "Probe latency is shown for comparison only and is not included in production log latency or success-rate calculations.": "探测延迟仅用于对比，不计入生产日志延迟或成功率计算。", + "Partial data": "部分数据", + "Weak correlation": "弱关联", + "Heuristic outcome": "启发式结果", + "Coarse latency": "粗粒度延迟", + "Probe data is separate": "探测数据独立展示", + "Known non-attempt logs excluded": "已排除已知非调用日志", + "Time range was limited": "时间范围已限制", + "Results truncated": "结果已截断", + "Retry marker backfill pending": "重试标记回填中", + "Retry metrics unavailable": "重试指标不可用", + "Consumption logs disabled": "消费日志未启用", + "Error logs disabled": "错误日志未启用", + "Error log collection is disabled": "错误日志采集未启用", + "Errors may be incomplete": "错误记录可能不完整", + "Recorded error logs": "已记录错误日志", + "The displayed error count includes only recorded logs and may be incomplete. Estimated success rate is unavailable while error log collection is disabled.": "当前显示的错误次数仅包含已记录日志，数据可能不完整；错误日志采集未启用期间无法计算估算成功率。", + "Channel metadata missing": "渠道元数据缺失", + "Probe unavailable": "无探测数据", + "Channel performance": "渠道性能", + "Channel performance details": "渠道性能详情", + "Channel performance is not loaded automatically": "渠道性能不会自动加载", + "No recorded channel logs": "没有已记录的渠道日志", + "Unable to load channel performance": "无法加载渠道性能", + "Model performance": "模型性能", + "Observed model log events": "观测模型日志事件", + "Unable to load model performance": "无法加载模型性能", + "Model performance is not loaded automatically": "模型性能不会自动加载", + "No recorded model logs": "没有已记录的模型日志", + "Model performance details": "模型性能详情", + "This legacy view aggregates estimated model outcomes from eligible Consume and Error logs. It is not a complete Relay Attempt trace, and logged latency remains accurate only to whole seconds.": "此旧版视图根据符合条件的 Consume 和 Error 日志汇总估算模型结果。它不是完整的 Relay Attempt 链路，日志延迟仅精确到整秒。", + "Model results are aggregated from eligible logs and are not a complete Relay Attempt trace.": "模型结果根据符合条件的日志汇总，并不是完整的 Relay Attempt 链路。", + "Inspect read-only production performance and operational evidence.": "查看只读生产性能和运维证据。", + "Throughput unavailable": "吞吐量不可用", + "Model throughput could not be read from performance metrics. Other log-based results remain available; check performance metrics collection and database availability.": "无法从性能指标中读取模型吞吐量。其他基于日志的结果仍可用；请检查性能指标采集和数据库可用性。", + "Unable to load model performance details": "无法加载模型性能详情", + "Check performance metrics collection and database availability, then try again.": "请检查性能指标采集和数据库可用性，然后重试。", + "Bucket-level estimate": "桶级估算", + "New performance samples are not being collected. Existing performance data may be incomplete.": "性能样本采集已停止，现有性能数据可能不完整。", + "No performance samples were recorded for this time range.": "该时间范围内未记录性能样本。", + "No throughput samples": "无吞吐量样本", + "Performance metrics collection is disabled": "性能指标采集已关闭", + "Performance metrics could not be queried. Log-based model results remain available; check the application database.": "无法查询性能指标。基于日志的模型结果仍可用；请检查应用数据库。", + "Throughput coverage: {{start}} – {{end}} ({{minutes}} minute buckets).": "吞吐量覆盖范围：{{start}} – {{end}}（{{minutes}} 分钟桶）。", + "Throughput coverage: {{start}} – {{end}}. Bucket granularity is unknown or mixed, so this range is approximate.": "吞吐量覆盖范围：{{start}} – {{end}}。桶粒度未知或混合，因此该范围为近似值。", + "Throughput query failed": "吞吐量查询失败", + "Last 24 hours": "最近 24 小时", + "Channel models and groups from eligible production logs": "来自符合条件生产日志的渠道模型和分组", + "Unable to load channel performance details": "无法加载渠道性能详情", + "No channel model performance": "没有渠道模型性能数据", + "No eligible production logs were recorded for this channel in the last 24 hours.": "最近 24 小时未记录该渠道的符合条件生产日志。", + "The detail table was truncated to {{count}} rows. The summary still covers all matching eligible logs.": "详情表已截断为 {{count}} 行，但摘要仍覆盖所有符合条件的匹配日志。", + "Throughput data is partial": "吞吐量数据不完整", + "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete.": "当前显示已存储和本地性能样本，但无法读取共享活动桶；最近的多节点吞吐量可能不完整。" } } diff --git a/web/default/src/routeTree.gen.ts b/web/default/src/routeTree.gen.ts index e0add2a9..7b8bfa48 100644 --- a/web/default/src/routeTree.gen.ts +++ b/web/default/src/routeTree.gen.ts @@ -50,6 +50,10 @@ import { Route as AuthenticatedKeysIndexRouteImport } from './routes/_authentica import { Route as AuthenticatedDashboardIndexRouteImport } from './routes/_authenticated/dashboard/index' import { Route as AuthenticatedChannelsIndexRouteImport } from './routes/_authenticated/channels/index' import { Route as AuthenticatedUsageLogsSectionRouteImport } from './routes/_authenticated/usage-logs/$section' +import { Route as AuthenticatedSmartOpsSystemInfoRouteImport } from './routes/_authenticated/smart-ops/system-info' +import { Route as AuthenticatedSmartOpsProductionPerformanceRouteImport } from './routes/_authenticated/smart-ops/production-performance' +import { Route as AuthenticatedSmartOpsModelPerformanceRouteImport } from './routes/_authenticated/smart-ops/model-performance' +import { Route as AuthenticatedSmartOpsChannelPerformanceRouteImport } from './routes/_authenticated/smart-ops/channel-performance' import { Route as AuthenticatedModelsSectionRouteImport } from './routes/_authenticated/models/$section' import { Route as AuthenticatedErrorsErrorRouteImport } from './routes/_authenticated/errors/$error' import { Route as AuthenticatedDashboardSectionRouteImport } from './routes/_authenticated/dashboard/$section' @@ -286,6 +290,30 @@ const AuthenticatedUsageLogsSectionRoute = path: '/usage-logs/$section', getParentRoute: () => AuthenticatedRouteRoute, } as any) +const AuthenticatedSmartOpsSystemInfoRoute = + AuthenticatedSmartOpsSystemInfoRouteImport.update({ + id: '/smart-ops/system-info', + path: '/smart-ops/system-info', + getParentRoute: () => AuthenticatedRouteRoute, + } as any) +const AuthenticatedSmartOpsProductionPerformanceRoute = + AuthenticatedSmartOpsProductionPerformanceRouteImport.update({ + id: '/smart-ops/production-performance', + path: '/smart-ops/production-performance', + getParentRoute: () => AuthenticatedRouteRoute, + } as any) +const AuthenticatedSmartOpsModelPerformanceRoute = + AuthenticatedSmartOpsModelPerformanceRouteImport.update({ + id: '/smart-ops/model-performance', + path: '/smart-ops/model-performance', + getParentRoute: () => AuthenticatedRouteRoute, + } as any) +const AuthenticatedSmartOpsChannelPerformanceRoute = + AuthenticatedSmartOpsChannelPerformanceRouteImport.update({ + id: '/smart-ops/channel-performance', + path: '/smart-ops/channel-performance', + getParentRoute: () => AuthenticatedRouteRoute, + } as any) const AuthenticatedModelsSectionRoute = AuthenticatedModelsSectionRouteImport.update({ id: '/models/$section', @@ -429,6 +457,10 @@ export interface FileRoutesByFullPath { '/dashboard/$section': typeof AuthenticatedDashboardSectionRoute '/errors/$error': typeof AuthenticatedErrorsErrorRoute '/models/$section': typeof AuthenticatedModelsSectionRoute + '/smart-ops/channel-performance': typeof AuthenticatedSmartOpsChannelPerformanceRoute + '/smart-ops/model-performance': typeof AuthenticatedSmartOpsModelPerformanceRoute + '/smart-ops/production-performance': typeof AuthenticatedSmartOpsProductionPerformanceRoute + '/smart-ops/system-info': typeof AuthenticatedSmartOpsSystemInfoRoute '/usage-logs/$section': typeof AuthenticatedUsageLogsSectionRoute '/channels/': typeof AuthenticatedChannelsIndexRoute '/dashboard/': typeof AuthenticatedDashboardIndexRoute @@ -488,6 +520,10 @@ export interface FileRoutesByTo { '/dashboard/$section': typeof AuthenticatedDashboardSectionRoute '/errors/$error': typeof AuthenticatedErrorsErrorRoute '/models/$section': typeof AuthenticatedModelsSectionRoute + '/smart-ops/channel-performance': typeof AuthenticatedSmartOpsChannelPerformanceRoute + '/smart-ops/model-performance': typeof AuthenticatedSmartOpsModelPerformanceRoute + '/smart-ops/production-performance': typeof AuthenticatedSmartOpsProductionPerformanceRoute + '/smart-ops/system-info': typeof AuthenticatedSmartOpsSystemInfoRoute '/usage-logs/$section': typeof AuthenticatedUsageLogsSectionRoute '/channels': typeof AuthenticatedChannelsIndexRoute '/dashboard': typeof AuthenticatedDashboardIndexRoute @@ -551,6 +587,10 @@ export interface FileRoutesById { '/_authenticated/dashboard/$section': typeof AuthenticatedDashboardSectionRoute '/_authenticated/errors/$error': typeof AuthenticatedErrorsErrorRoute '/_authenticated/models/$section': typeof AuthenticatedModelsSectionRoute + '/_authenticated/smart-ops/channel-performance': typeof AuthenticatedSmartOpsChannelPerformanceRoute + '/_authenticated/smart-ops/model-performance': typeof AuthenticatedSmartOpsModelPerformanceRoute + '/_authenticated/smart-ops/production-performance': typeof AuthenticatedSmartOpsProductionPerformanceRoute + '/_authenticated/smart-ops/system-info': typeof AuthenticatedSmartOpsSystemInfoRoute '/_authenticated/usage-logs/$section': typeof AuthenticatedUsageLogsSectionRoute '/_authenticated/channels/': typeof AuthenticatedChannelsIndexRoute '/_authenticated/dashboard/': typeof AuthenticatedDashboardIndexRoute @@ -613,6 +653,10 @@ export interface FileRouteTypes { | '/dashboard/$section' | '/errors/$error' | '/models/$section' + | '/smart-ops/channel-performance' + | '/smart-ops/model-performance' + | '/smart-ops/production-performance' + | '/smart-ops/system-info' | '/usage-logs/$section' | '/channels/' | '/dashboard/' @@ -672,6 +716,10 @@ export interface FileRouteTypes { | '/dashboard/$section' | '/errors/$error' | '/models/$section' + | '/smart-ops/channel-performance' + | '/smart-ops/model-performance' + | '/smart-ops/production-performance' + | '/smart-ops/system-info' | '/usage-logs/$section' | '/channels' | '/dashboard' @@ -734,6 +782,10 @@ export interface FileRouteTypes { | '/_authenticated/dashboard/$section' | '/_authenticated/errors/$error' | '/_authenticated/models/$section' + | '/_authenticated/smart-ops/channel-performance' + | '/_authenticated/smart-ops/model-performance' + | '/_authenticated/smart-ops/production-performance' + | '/_authenticated/smart-ops/system-info' | '/_authenticated/usage-logs/$section' | '/_authenticated/channels/' | '/_authenticated/dashboard/' @@ -1075,6 +1127,34 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof AuthenticatedUsageLogsSectionRouteImport parentRoute: typeof AuthenticatedRouteRoute } + '/_authenticated/smart-ops/system-info': { + id: '/_authenticated/smart-ops/system-info' + path: '/smart-ops/system-info' + fullPath: '/smart-ops/system-info' + preLoaderRoute: typeof AuthenticatedSmartOpsSystemInfoRouteImport + parentRoute: typeof AuthenticatedRouteRoute + } + '/_authenticated/smart-ops/production-performance': { + id: '/_authenticated/smart-ops/production-performance' + path: '/smart-ops/production-performance' + fullPath: '/smart-ops/production-performance' + preLoaderRoute: typeof AuthenticatedSmartOpsProductionPerformanceRouteImport + parentRoute: typeof AuthenticatedRouteRoute + } + '/_authenticated/smart-ops/model-performance': { + id: '/_authenticated/smart-ops/model-performance' + path: '/smart-ops/model-performance' + fullPath: '/smart-ops/model-performance' + preLoaderRoute: typeof AuthenticatedSmartOpsModelPerformanceRouteImport + parentRoute: typeof AuthenticatedRouteRoute + } + '/_authenticated/smart-ops/channel-performance': { + id: '/_authenticated/smart-ops/channel-performance' + path: '/smart-ops/channel-performance' + fullPath: '/smart-ops/channel-performance' + preLoaderRoute: typeof AuthenticatedSmartOpsChannelPerformanceRouteImport + parentRoute: typeof AuthenticatedRouteRoute + } '/_authenticated/models/$section': { id: '/_authenticated/models/$section' path: '/models/$section' @@ -1301,6 +1381,10 @@ interface AuthenticatedRouteRouteChildren { AuthenticatedDashboardSectionRoute: typeof AuthenticatedDashboardSectionRoute AuthenticatedErrorsErrorRoute: typeof AuthenticatedErrorsErrorRoute AuthenticatedModelsSectionRoute: typeof AuthenticatedModelsSectionRoute + AuthenticatedSmartOpsChannelPerformanceRoute: typeof AuthenticatedSmartOpsChannelPerformanceRoute + AuthenticatedSmartOpsModelPerformanceRoute: typeof AuthenticatedSmartOpsModelPerformanceRoute + AuthenticatedSmartOpsProductionPerformanceRoute: typeof AuthenticatedSmartOpsProductionPerformanceRoute + AuthenticatedSmartOpsSystemInfoRoute: typeof AuthenticatedSmartOpsSystemInfoRoute AuthenticatedUsageLogsSectionRoute: typeof AuthenticatedUsageLogsSectionRoute AuthenticatedChannelsIndexRoute: typeof AuthenticatedChannelsIndexRoute AuthenticatedDashboardIndexRoute: typeof AuthenticatedDashboardIndexRoute @@ -1324,6 +1408,13 @@ const AuthenticatedRouteRouteChildren: AuthenticatedRouteRouteChildren = { AuthenticatedDashboardSectionRoute: AuthenticatedDashboardSectionRoute, AuthenticatedErrorsErrorRoute: AuthenticatedErrorsErrorRoute, AuthenticatedModelsSectionRoute: AuthenticatedModelsSectionRoute, + AuthenticatedSmartOpsChannelPerformanceRoute: + AuthenticatedSmartOpsChannelPerformanceRoute, + AuthenticatedSmartOpsModelPerformanceRoute: + AuthenticatedSmartOpsModelPerformanceRoute, + AuthenticatedSmartOpsProductionPerformanceRoute: + AuthenticatedSmartOpsProductionPerformanceRoute, + AuthenticatedSmartOpsSystemInfoRoute: AuthenticatedSmartOpsSystemInfoRoute, AuthenticatedUsageLogsSectionRoute: AuthenticatedUsageLogsSectionRoute, AuthenticatedChannelsIndexRoute: AuthenticatedChannelsIndexRoute, AuthenticatedDashboardIndexRoute: AuthenticatedDashboardIndexRoute, diff --git a/web/default/src/routes/_authenticated/smart-ops/channel-performance.tsx b/web/default/src/routes/_authenticated/smart-ops/channel-performance.tsx new file mode 100644 index 00000000..9204438d --- /dev/null +++ b/web/default/src/routes/_authenticated/smart-ops/channel-performance.tsx @@ -0,0 +1,34 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import { createFileRoute, redirect } from '@tanstack/react-router' +import { useAuthStore } from '@/stores/auth-store' +import { ROLE } from '@/lib/roles' +import { ProductionPerformance } from '@/features/smart-ops' + +export const Route = createFileRoute( + '/_authenticated/smart-ops/channel-performance' +)({ + beforeLoad: () => { + const { auth } = useAuthStore.getState() + if (!auth.user || auth.user.role < ROLE.ADMIN) { + throw redirect({ to: '/403' }) + } + }, + component: ProductionPerformance, +}) diff --git a/web/default/src/routes/_authenticated/smart-ops/model-performance.tsx b/web/default/src/routes/_authenticated/smart-ops/model-performance.tsx new file mode 100644 index 00000000..b0313947 --- /dev/null +++ b/web/default/src/routes/_authenticated/smart-ops/model-performance.tsx @@ -0,0 +1,34 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import { createFileRoute, redirect } from '@tanstack/react-router' +import { useAuthStore } from '@/stores/auth-store' +import { ROLE } from '@/lib/roles' +import { ModelPerformance } from '@/features/smart-ops/model-performance' + +export const Route = createFileRoute( + '/_authenticated/smart-ops/model-performance' +)({ + beforeLoad: () => { + const { auth } = useAuthStore.getState() + if (!auth.user || auth.user.role < ROLE.ADMIN) { + throw redirect({ to: '/403' }) + } + }, + component: ModelPerformance, +}) diff --git a/web/default/src/routes/_authenticated/smart-ops/production-performance.tsx b/web/default/src/routes/_authenticated/smart-ops/production-performance.tsx new file mode 100644 index 00000000..526304b9 --- /dev/null +++ b/web/default/src/routes/_authenticated/smart-ops/production-performance.tsx @@ -0,0 +1,27 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import { createFileRoute, redirect } from '@tanstack/react-router' + +export const Route = createFileRoute( + '/_authenticated/smart-ops/production-performance' +)({ + beforeLoad: () => { + throw redirect({ to: '/smart-ops/channel-performance' }) + }, +}) diff --git a/web/default/src/routes/_authenticated/smart-ops/system-info.tsx b/web/default/src/routes/_authenticated/smart-ops/system-info.tsx new file mode 100644 index 00000000..3401eafb --- /dev/null +++ b/web/default/src/routes/_authenticated/smart-ops/system-info.tsx @@ -0,0 +1,32 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import { createFileRoute, redirect } from '@tanstack/react-router' +import { useAuthStore } from '@/stores/auth-store' +import { ROLE } from '@/lib/roles' +import { SystemInfo } from '@/features/system-info' + +export const Route = createFileRoute('/_authenticated/smart-ops/system-info')({ + beforeLoad: () => { + const { auth } = useAuthStore.getState() + if (auth.user?.role !== ROLE.SUPER_ADMIN) { + throw redirect({ to: '/403' }) + } + }, + component: SystemInfo, +}) diff --git a/web/default/src/routes/_authenticated/system-info/index.tsx b/web/default/src/routes/_authenticated/system-info/index.tsx index be79abac..5c512153 100644 --- a/web/default/src/routes/_authenticated/system-info/index.tsx +++ b/web/default/src/routes/_authenticated/system-info/index.tsx @@ -19,7 +19,6 @@ For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API import { createFileRoute, redirect } from '@tanstack/react-router' import { useAuthStore } from '@/stores/auth-store' import { ROLE } from '@/lib/roles' -import { SystemInfo } from '@/features/system-info' export const Route = createFileRoute('/_authenticated/system-info/')({ beforeLoad: () => { @@ -30,6 +29,7 @@ export const Route = createFileRoute('/_authenticated/system-info/')({ to: '/403', }) } + + throw redirect({ to: '/smart-ops/system-info' }) }, - component: SystemInfo, }) From 0644df24df6ba2229534d6b3a1b965740bca39f1 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Thu, 20 Aug 2026 02:24:33 +0800 Subject: [PATCH 02/28] add features --- ...76\345\214\272\344\273\213\347\273\215.md" | 237 ---------- README_preview.md | 417 ------------------ controller/channel_performance.go | 32 +- controller/model_performance.go | 32 +- controller/performance_error.go | 24 + docker-compose.yml | 2 +- model/perf_metric.go | 36 +- model/perf_metric_test.go | 43 ++ pkg/perf_metrics/flush.go | 10 +- pkg/perf_metrics/metrics.go | 170 +++++-- pkg/perf_metrics/metrics_test.go | 192 +++++++- service/channel_performance.go | 51 +-- service/channel_performance_test.go | 8 +- service/model_performance.go | 57 +-- service/model_performance_test.go | 7 +- service/performance_query.go | 67 +++ .../src/features/performance-metrics/types.ts | 2 +- .../components/model-details-chart-utils.ts | 38 +- .../components/model-details-charts.tsx | 18 +- .../components/model-details-performance.tsx | 30 +- .../model-details-uptime-sparkline.tsx | 30 +- web/default/src/features/smart-ops/api.ts | 5 +- .../src/features/smart-ops/index.test.tsx | 15 - web/default/src/features/smart-ops/index.tsx | 116 +++-- .../smart-ops/lib/channel-detail.test.ts | 76 ++++ .../features/smart-ops/lib/channel-detail.ts | 8 +- .../features/smart-ops/lib/filters.test.ts | 27 ++ .../src/features/smart-ops/lib/filters.ts | 3 +- .../src/features/smart-ops/lib/model-sort.ts | 2 +- .../src/features/smart-ops/lib/sort.test.ts | 1 - .../src/features/smart-ops/lib/sort.ts | 4 +- .../smart-ops/model-performance.test.tsx | 73 ++- .../features/smart-ops/model-performance.tsx | 243 ++++++---- web/default/src/features/smart-ops/types.ts | 3 +- web/default/src/i18n/locales/ru.json | 6 +- web/default/src/i18n/locales/vi.json | 4 +- web/default/src/i18n/locales/zh.json | 2 +- 37 files changed, 1050 insertions(+), 1041 deletions(-) delete mode 100644 "MAX_API_Next\347\244\276\345\214\272\344\273\213\347\273\215.md" delete mode 100644 README_preview.md create mode 100644 controller/performance_error.go create mode 100644 model/perf_metric_test.go create mode 100644 service/performance_query.go create mode 100644 web/default/src/features/smart-ops/lib/channel-detail.test.ts diff --git "a/MAX_API_Next\347\244\276\345\214\272\344\273\213\347\273\215.md" "b/MAX_API_Next\347\244\276\345\214\272\344\273\213\347\273\215.md" deleted file mode 100644 index 0b381193..00000000 --- "a/MAX_API_Next\347\244\276\345\214\272\344\273\213\347\273\215.md" +++ /dev/null @@ -1,237 +0,0 @@ -# MAX API Next：面向 AI Models 与 Agents 治理的新一代开源社区 - -当 AI 应用从 Demo 走向真实业务，问题很快就不再只是“能不能调用一个模型”。 - -越来越多的团队开始同时接入 OpenAI、Claude、Gemini、Azure、AWS、Vertex、Ollama，以及 DeepSeek、通义千问、智谱 GLM、Kimi、豆包、混元、文心、讯飞星火、MiniMax 等国内外模型与平台。与此同时，Agent 工作流开始接管更多复杂任务：它们会连续调用模型、工具、知识库、图像、视频、音频和搜索能力，也会产生更长链路、更高成本和更复杂的权限边界。 - -这意味着，AI 应用时代真正需要被系统化解决的问题，已经从“模型调用”扩展为“模型与 Agent 的持续治理”。 - -MAX API Next 社区正是在这个背景下成立的。**社区由来自科研机构和高校的 AGI 爱好者组织发起、维护和运营，是一个研究驱动的 AI Models 与 Agents 治理项目**，长期关注模型平台适配、AgentOps 工程实践、成本审计、治理边界和 AI 应用基础设施建设。 - - - -![在这里插入图片描述](https://i-blog.csdnimg.cn/direct/9222cf388dec4deb826923061623ddb3.jpeg) - -## MAX API Next 是什么？ - -MAX API Next 是一个聚焦 **AI Models 与 Agents 治理** 的技术社区。 - -社区围绕 AI 应用落地后的基础设施问题，持续建设可复用的工程能力： - -- 如何统一接入不同模型平台与协议？ -- 如何让应用和 Agent 使用稳定、可控的模型入口？ -- 如何管理 API Key、用户、组织、模型范围和额度？ -- 如何在多供应商之间做路由、降级、重试与成本控制？ -- 如何让模型价格、任务价格和 Agent 调用成本更直观？ -- 如何在私有化部署中保留审计、日志、权限与安全边界？ -- 如何持续跟踪国产模型、国际平台和多模态接口的快速变化？ - -我们希望 MAX API Next 成为一个面向开发者、研究者、团队和组织的 AI 治理社区：把上游模型服务、Agent 应用、用户体系、成本审计和运营控制连接在一起，让 AI 应用可以更稳定、更可控、更可持续地运行。 - -## 为什么现在需要 AI Models 与 Agents 治理？ - -过去，很多 AI 应用只需要一个模型 Key、一段调用代码和一个简单的代理服务。 - -但今天的 AI 应用已经明显不同： - -- 模型供应商越来越多，接口格式和参数差异频繁变化。 -- 国产模型和平台迭代很快，价格、能力、上下文、限流策略不断调整。 -- Agent 不再是单次问答，而是长链路、多步骤、多工具、多模型组合调用。 -- 多模态能力快速进入生产环境，图像、视频、音频、embedding、rerank、实时语音都需要统一管理。 -- 成本核算从“按 token”扩展到“按模型、按任务、按质量、按时长、按阶段、按供应商规则”。 -- 企业、团队和社区服务需要更清晰的用户、令牌、权限、日志和审计边界。 - -当这些问题叠加在一起，一个统一的治理层就变得非常关键。 - -MAX API Next 社区希望推动的方向，就是把 AI Models 和 Agents 的接入、路由、计费、权限、审计和运营能力沉淀为基础设施，而不是让每个应用重复实现一遍。 - -## MAX API：AGI 应用服务基础设施 -![在这里插入图片描述](https://i-blog.csdnimg.cn/direct/7d87f76424c74e31b4100ff7c97a9815.jpeg) - -MAX API 是 MAX API Next 社区中的核心开源项目之一。 - -它可以理解为一个面向 AI Models 与 Agents 的 API 网关、运营中台和治理层，位于应用、Agent、用户、组织和上游模型服务商之间。 - -应用侧可以通过统一接口访问不同模型能力；管理侧可以统一配置渠道、模型、价格、分组、令牌、日志、路由和安全策略；运营侧可以观察消耗、成本、错误、任务状态和使用情况。 - -![在这里插入图片描述](https://i-blog.csdnimg.cn/direct/8f965eed98c74f358b0cb9f61e93c00c.png) - -更具体地说，MAX API 希望解决三类核心问题： - -1. **模型接入问题**：多模型、多平台、多协议、多模态能力如何统一接入。 -2. **AgentOps 问题**：Agent 的令牌、权限、成本、调用链路和异常如何治理。 -3. **运营治理问题**：渠道、价格、日志、审计、安全和私有化部署如何长期维护。 -## 核心能力一：统一 AI 模型入口 - -MAX API 支持将不同类型的模型接口纳入统一治理，包括但不限于： - -- OpenAI-compatible 接口 -- OpenAI Responses -- Claude Messages -- Gemini -- Azure OpenAI -- AWS Bedrock -- Vertex AI -- Ollama -- embeddings -- rerank -- images -- audio -- realtime -- video tasks -- model discovery - -对于应用开发者来说，这意味着应用侧不必为每一家供应商维护一套完整的接入逻辑。对于团队和组织来说，这意味着模型入口、鉴权、路由和成本可以被统一管理。 - - -## 核心能力二：持续适配国产模型与平台 - -MAX API 特别重视国内模型和平台生态的持续跟踪适配。 - -国内 AI 平台的模型更新、接口变化、价格调整和多模态能力发布节奏很快。如果每个团队都单独跟进，会产生大量重复工作。 - -MAX API 将这类变化沉淀到统一的渠道配置、模型映射、协议转换、价格规则和任务协议中，帮助开发者更快接入和切换不同模型平台。 - -这不仅包括文本模型，也包括图像、视频、音频、embedding、rerank、Agent 应用平台和工作流平台等更广泛的 AI 能力生态。 - - -![在这里插入图片描述](https://i-blog.csdnimg.cn/direct/9b12f6b5f3f54509aad84c7b02b39d2b.png) - -## 核心能力三：面向 Agent 的治理能力 - -Agent 应用的复杂度通常高于普通聊天应用。 - -一个 Agent 可能会在一次任务中连续调用多个模型、多种工具和多个外部服务。如果缺少治理层，就很难回答这些问题： - -- 这个 Agent 能调用哪些模型？ -- 它是否应该使用高成本模型？ -- 它的额度是否需要单独限制？ -- 它的调用失败发生在哪个上游渠道？ -- 它的成本应该归因到哪个用户、团队或业务线？ -- 出现异常时，管理员能否快速定位问题？ - -MAX API 围绕 AgentOps 提供令牌隔离、模型范围控制、额度限制、渠道路由、失败重试、日志记录、成本统计和审计边界等能力，让 Agent 应用在真实业务中更容易被运营和管理。 - -## 核心能力四：更直观的计费与成本审计 - -AI 成本正在变得越来越复杂。 - -传统的“倍率计费”已经很难覆盖真实场景。不同模型可能存在输入 token、输出 token、缓存命中、图像 token、音频 token、分阶段价格、固定任务价格、视频时长、清晰度、是否带音频、是否有参考图等多种计费因素。 - -MAX API 在计费方向持续优化，重点包括： - -- 支持更直观的价格表达式。 -- 支持按真实价格维护模型计费规则。 -- 支持分阶段计费 JSON 批量配置。 -- 支持视频等异步任务的 rate-card 规则。 -- 支持按用户、令牌、模型、渠道和分组统计用量。 -- 支持预扣费、失败退款和任务完成后的成本归因。 - -对于运营者来说，计费配置不应该是一堆难以理解的倍率参数，而应该尽量接近上游价格表本身，便于核对、更新和审计。 - -## 核心能力五：渠道能力矩阵与配置校验 - -多渠道治理的难点之一，是管理员很难直观看到某个渠道到底支持哪些能力，也很难在保存配置前发现错误。 - -MAX API 正在强化渠道配置体验，例如： - -- 展示渠道能力矩阵。 -- 标识 chat/completions、responses、embeddings、rerank、images、audio、video tasks 等能力。 -- 对 Base URL、API Key、模型列表、JSON 配置、Vertex 区域、Codex 凭证、视频任务路径占位符等常见问题进行提示。 -- 通过更清晰的表单和校验减少误配置。 - -这类能力看起来不像模型本身那么显眼，但对长期运营非常重要。因为真实系统里的很多故障，往往不是模型不可用，而是渠道配置错误、协议路径变化、价格规则遗漏或权限边界不清。 - - -![在这里插入图片描述](https://i-blog.csdnimg.cn/direct/44c30d84ccf9403bba732cc4c412433b.png) -## 核心能力六：通用视频任务协议 - -视频生成、图像生成、长任务推理等异步任务越来越常见。 - -这类接口通常不是一次请求直接返回结果，而是经历： - -1. 提交任务。 -2. 获得任务 ID。 -3. 轮询任务状态。 -4. 解析进度、错误和结果。 -5. 返回代理后的内容地址。 -6. 按任务参数计费。 - -MAX API 将这类能力抽象为可配置的视频任务协议，支持任务提交路径、查询路径、状态映射、进度路径、错误消息路径、结果路径和内容代理等配置。 - -这样，不同上游平台即使接口路径和返回结构不同，也可以通过统一的任务协议纳入治理。 - -## 核心能力七：私有化部署与安全边界 - -MAX API 支持自托管部署，适合需要自主掌控密钥、数据、日志和策略的团队。 - -在私有化场景中，系统可以承担统一入口、用户管理、令牌管理、分组权限、模型范围、渠道路由、请求日志、错误日志、使用统计和安全限制等职责。 - -对于涉及输入输出内容记录的场景，MAX API 也强调审计边界：这类能力应当放在管理员端、安全与限制相关配置中，并在具备合法依据、明确告知、权限隔离和数据安全措施的前提下谨慎启用。 - - -![在这里插入图片描述](https://i-blog.csdnimg.cn/direct/aee0bc322cbb48909f94739f1ef948c7.png) -## MAX API 适合谁？ - -MAX API 更适合以下几类使用者： - -- 正在维护多个模型供应商接入的开发者。 -- 需要统一管理模型 Key、用户、分组和额度的团队。 -- 正在建设 Agent 应用、工作流应用或内部 AI 平台的组织。 -- 需要接入国产模型、国际模型和多模态能力的项目。 -- 需要对 AI 调用成本进行统计、审计和归因的运营团队。 -- 需要私有化部署 AI API 网关的研究机构、高校团队和企业团队。 - -如果你的应用只调用一个固定模型，并且没有用户、权限、成本和路由管理需求，那么一个简单的 SDK 可能就够了。 - -但如果你正在面对多模型、多供应商、多 Agent、多用户、多计费规则和多模态任务，MAX API 这类治理层就会变得越来越有价值。 - -## 社区希望一起建设什么？ - -MAX API Next 社区希望和更多开发者一起建设 AI Models 与 Agents 治理生态。 - -我们关注的问题包括： - -- 新模型与新平台的持续适配。 -- 国产模型生态的接口、价格和能力跟踪。 -- AgentOps 的工程实践与治理边界。 -- 多模态任务协议抽象。 -- 更直观的模型计费与成本审计。 -- 私有化部署、日志审计、安全限制和合规提示。 -- 面向真实生产环境的稳定性、可观测性和运维体验。 - -AI 应用的发展速度很快，但真正决定它能否长期运行的，往往是那些不在演示视频里的基础能力：权限、成本、路由、审计、配置、日志、任务状态、失败处理和持续维护。 - -MAX API Next 社区希望把这些能力做好，并把它们开放给更多正在建设 AI 应用和 Agent 系统的人。 - -## 结语 - -AI Models 是能力源头，Agents 是应用形态，而治理层决定了这些能力能否稳定进入真实系统。 - -MAX API Next 社区将继续围绕 AI Models 与 Agents 治理，推进模型接入、国产平台适配、AgentOps、计费审计、多模态任务和私有化运营能力建设。 - -MAX API 是这个方向上的一个重要实践。它不是要替代模型厂商，也不是要替代 Agent 编排框架，而是希望成为应用、Agent、用户、组织和上游模型服务之间稳定、可控、可运营的基础设施层。 - -欢迎关注 MAX API Next 社区，一起建设面向 AI 应用时代的 Models 与 Agents 治理基础设施。 - ---- - -**社区访问** - -GitHub地址：https://github.com/MAX-API-Next -社群交流：QQ(950126533) | 微信群(搜索MAX-API) - -**关键词** - -AI API Gateway / AI Models Governance / AgentOps / OpenAI-compatible / 国产模型适配 / 多模型路由 / 成本审计 / rate-card / 私有化部署 / 多模态任务治理 - -**合规提示** - -在面向公众提供生成式 AI 服务、商业化运营或接入第三方上游模型服务时，请自行完成所在司法辖区要求的备案、授权、内容安全、实名、日志留存、税务、支付、隐私保护和用户协议等合规事项。 - -**相关文献/项目** - -| [One API](https://github.com/songquanpeng/one-api) | MIT 协议 | -| [New API](https://github.com/QuantumNous/new-api) | AGPLv3 协议 | -| [Midjourney-Proxy](https://github.com/novicezk/midjourney-proxy) | Apache-2.0 协议 | -| [Suno API](https://github.com/Suno-API/Suno-API) | MIT 协议 | \ No newline at end of file diff --git a/README_preview.md b/README_preview.md deleted file mode 100644 index bb7c8fa2..00000000 --- a/README_preview.md +++ /dev/null @@ -1,417 +0,0 @@ -
- -![MAX API](./web/default/public/logo.png) - -# MAX API - -**AI Models 与 Agents 的统一接入、治理和运营基础设施** - -把多供应商模型、Agent 工作负载、成本、路由、日志和权限放进同一套可控边界。 - -

- 简体中文 | - 繁體中文 | - English | - Français | - 日本語 -

- -

- - license - - release - - docker - - GoReportCard - -

- -

- 快速开始 • - 核心能力 • - 生产部署 • - 常见问题 • - 资源与社区 -

- -
- ---- - -MAX API 是由来自科研机构和高校的 AGI 爱好者组织发起、维护和长期运营的开源项目。它位于应用、Agent 与上游模型服务之间，提供统一接口、身份鉴权、渠道路由、计费结算、日志观测和组织治理能力。 - -它不替代 Dify、LangChain、MCP Server 或业务 Agent，也不提供上游模型账号和 API Key。MAX API 的职责，是帮助已有 AI 应用和 Agent 更稳定、更可控地进入真实运行环境。 - -> [!IMPORTANT] -> 面向公众提供生成式人工智能服务时，使用者应自行完成所在司法辖区要求的上游授权、备案许可、内容安全、实名、日志留存、税务、支付和用户协议等合规义务。日志审计和内容留存仅应在具备合法依据、明确告知、权限隔离和数据安全措施的场景下启用。 - -> [!TIP] -> 生产环境建议使用正式版。Preview 版本用于测试和灰度验证，升级前请备份数据库并准备回滚方案。 - -## 🚀 快速开始 - -本地体验默认使用 SQLite，只需 Docker： - -```bash -docker pull cscitechtop/max-api:latest - -docker run --name max-api -d --restart always -p 3000:3000 -e TZ=Asia/Shanghai -v ./data:/data cscitechtop/max-api:latest -``` - -启动后访问： - -接下来只需完成三件事： - -1. 创建或确认管理员账号。 -2. 添加一个具有合法授权的上游渠道和 API Key。 -3. 创建访问令牌，在应用中把 Base URL 指向 MAX API。 - -> [!WARNING] -> SQLite 适合本地体验、开发和小规模测试。正式环境建议使用 MySQL ≥ 5.7.8 或 PostgreSQL ≥ 9.6，并配置 Redis、HTTPS、备份和恢复方案。 - -## ✨ 核心能力 - -| 能力 | MAX API 解决的问题 | -|---|---| -| 统一模型入口 | 用统一网关接入 OpenAI Compatible、Responses、Claude Messages、Gemini、Realtime 和多模态任务接口 | -| 多供应商路由 | 管理渠道、权重、分组、模型映射、失败重试和跨供应商切换，降低单一上游风险 | -| Agent 治理 / AgentOps | 为 Agent、工作流和工具调用分配独立令牌、模型范围、额度与调用记录 | -| 成本与计费 | 支持倍率、固定价格、表达式计费、异步任务 rate-card、预扣费、结算和失败退款 | -| 可观测与审计 | 按用户、令牌、模型、渠道、分组和节点查看用量、耗时、错误、重试与管理员审计信息 | -| 私有化与扩展 | 支持 SQLite、MySQL、PostgreSQL、Redis、多节点部署和自定义上游协议配置 | - -### 使用 MAX API 的优势 - -| 维度 | 直接连接多个供应商 | 使用 MAX API | -|---|---|---| -| 应用接入 | 每家 SDK、协议和错误格式分别维护 | 应用侧使用相对稳定的统一入口 | -| 模型切换 | 需要修改代码和配置 | 通过渠道、映射、分组和路由调整 | -| 成本核算 | 账单分散，难以归因 | 按用户、令牌、模型、渠道和节点统一统计 | -| 权限治理 | 凭据和权限散落在不同应用 | 集中管理令牌、模型范围、额度和有效期 | -| 故障排查 | 日志分散，跨供应商关联困难 | 在网关侧统一查看请求、重试、耗时和错误 | - -## 🎯 适用场景 - -- **团队或组织内部模型网关**：统一管理用户、令牌、模型、供应商、权限和费用。 -- **Agent 应用运行底座**：为 Agent、工作流和工具调用提供模型访问控制、成本归因和异常定位。 -- **多供应商容灾与迁移**：在不同模型平台之间进行映射、加权路由、失败重试和灰度切换。 -- **多模态任务治理**：统一管理图像、音频、视频、嵌入、重排序和实时对话接口。 -- **私有化与合规运营**：自主管理密钥、数据、日志、审计、价格和部署环境。 - -## 🧭 工作方式 - -![MAX API 系统架构图](./docs/images/MAX-API架构图.png) - -一次模型请求通常经过以下链路： - -```text -应用 / SDK / Agent - → 统一接口与身份鉴权 - → 模型权限、限流和安全检查 - → 渠道选择、映射与失败重试 - → 上游协议适配 - → 用量结算、日志和审计 -``` - -MAX API 保持分层架构：`Router → Controller → Service → Model`，供应商协议转换位于 `relay/` 与 `relay/channel/`。数据层使用 GORM，兼容 SQLite、MySQL 和 PostgreSQL；Redis 与内存缓存用于限流、会话和多节点协同。 - -
-开发者技术栈 - -| 层 | 技术 | -|---|---| -| 后端 | Go 1.25.1+、Gin、GORM v2 | -| 前端 | React 19、TypeScript、Rsbuild、Base UI、Tailwind CSS | -| 前端包管理 | Bun workspace | -| 数据库 | SQLite / MySQL ≥ 5.7.8 / PostgreSQL ≥ 9.6 | -| 缓存 | Redis + 内存缓存 | -| 鉴权 | JWT、WebAuthn/Passkeys、OAuth、OIDC | - -
- -## 🤖 模型与接口 - -> 实际可用能力取决于你的上游授权、渠道配置、模型映射和供应商支持。MAX API 负责治理这些能力，不提供模型服务本身。 - -### 主要接口 - -| 类别 | 接口或能力 | -|---|---| -| Chat / Responses | `/v1/chat/completions`、`/v1/responses`，以及 Responses ↔ Chat Completions 兼容转换 | -| Embeddings / Rerank | `/v1/embeddings`、`/v1/rerank` | -| Images / Audio / Video | `/v1/images/*`、`/v1/audio/*`、`/v1/videos/*` | -| 原生协议 | Claude Messages、Google Gemini、OpenAI Realtime 等入口 | -| 异步任务 | 任务提交、轮询、状态映射、结果代理和参数化计费 | -| 自定义上游 | Base URL、路径覆盖、参数覆盖、Header 覆盖、状态和结果字段映射 | - -### 供应商与生态 - -可接入 OpenAI、Azure OpenAI、Claude、Gemini、AWS Bedrock、Vertex AI、Ollama，以及 DeepSeek、通义千问 / 阿里云百炼、智谱 GLM、Kimi、豆包 / 火山引擎、腾讯混元、百度文心 / 千帆、讯飞星火、MiniMax、零一万物、硅基流动等平台。 - -同时支持 Codex、Dify、RAGFlow、Kling、Seedance、Midjourney、Suno 等应用、Agent 或多模态服务相关的接入治理。具体能力以渠道类型和当前版本为准。 - -
-Reasoning Effort 模型命名示例 - -- OpenAI：`o3-mini-high`、`gpt-5-medium`、`gpt-5-low` -- Claude：`claude-3-7-sonnet-20250219-thinking` -- Gemini：`gemini-2.5-flash-thinking`、`gemini-2.5-pro-thinking-128` - -
- -## 🛡️ 治理与运营 - -推荐按照下面的顺序配置： - -1. 配置登录、安全限制和用户注册策略。 -2. 添加上游渠道，确认 Base URL、模型列表、能力和合法授权凭据。 -3. 按团队或业务线设置用户分组、令牌分组和模型范围。 -4. 配置模型价格、任务计费、额度、失败重试和日志策略。 -5. 为每个应用、Agent 或环境创建独立令牌。 -6. 通过使用日志、数据看板和渠道测试持续观察运行状态。 - -| 管理入口 | 用途 | -|---|---| -| 渠道管理 | 维护供应商、模型映射、权重、协议路径、能力矩阵和配置校验 | -| 模型与价格 | 维护模型列表、倍率、固定价格、表达式计费和任务 rate-card | -| 令牌管理 | 为应用、Agent、工作流和用户限制模型范围、额度和有效期 | -| 使用日志 | 查看消耗、耗时、错误、重试、渠道命中和审计信息 | -| 系统设置 | 管理安全、性能、日志、运营、计费、支付和站点配置 | -| 数据看板 | 查看请求量、模型用量、消费趋势和渠道运行状态 | - -
-高级治理能力 - -- **渠道能力矩阵与校验**：识别 `chat/completions`、`responses`、`embeddings`、`rerank`、视频任务和模型发现能力，并提示 Base URL、JSON、Vertex AI、Codex 凭据及任务路径配置风险。 -- **通用视频任务协议**：可配置提交路径、查询路径、任务 ID、状态、进度、结果 URL 和错误字段映射；请求体透传使用 `Pass Through Body`，字段改写使用 `Param Override`。 -- **计费 JSON 维护**：支持批量维护表达式计费和任务 rate-card；视频任务可按时长、清晰度、音频等请求参数匹配价格。 -- **管理员日志审计**：普通用户日志接口会过滤管理员专用字段；启用内容留存前应确认合规依据和访问权限。 -- **性能治理**：支持模型限流、流式超时、大响应缓冲、请求体限制、磁盘缓存、Pyroscope 和优雅关闭。 - -完整配置参考请查看 [详细中文文档](./README.zh_CN.md)。 - -
- -## 🚢 生产部署 - -推荐使用 Docker Compose： - -```bash -git clone https://github.com/MAX-API-Next/MAX-API.git -cd MAX-API - -# 修改 docker-compose.yml 中的数据库、Redis 密码和密钥 -docker compose up -d -``` - -### 部署要求 - -| 组件 | 建议 | -|---|---| -| 数据库 | MySQL ≥ 5.7.8 或 PostgreSQL ≥ 9.6，并配置备份与恢复 | -| 缓存 | 单机可使用内存缓存，多机部署建议使用 Redis | -| 入口 | HTTPS 反向代理、请求大小限制和可信网络策略 | -| 密钥 | 显式配置随机 `SESSION_SECRET`；Redis/多节点场景配置统一 `CRYPTO_SECRET` | -| 节点 | 多节点设置稳定且唯一的 `NODE_NAME` | -| 日志 | 根据合规和运维需要配置日志目录、LOG_DB、清理与保留策略 | - -
-常用环境变量 - -| 变量 | 用途 | 默认值 | -|---|---|---| -| `SQL_DSN` | 主数据库连接 | SQLite | -| `LOG_SQL_DSN` | 可选独立日志数据库 | 与主数据库相同 | -| `REDIS_CONN_STRING` | Redis 连接 | - | -| `SESSION_SECRET` | 会话密钥，多机必须一致 | - | -| `CRYPTO_SECRET` | 加密密钥，Redis/多机必须一致 | - | -| `NODE_NAME` | 节点名称与日志归属 | - | -| `STREAMING_TIMEOUT` | 流式响应超时，单位秒 | `300` | -| `STREAM_SCANNER_MAX_BUFFER_MB` | 流式单行最大缓冲 | `64` | -| `MAX_REQUEST_BODY_MB` | 解压后请求体上限 | `32` | -| `ERROR_LOG_ENABLED` | 错误请求日志开关 | `false` | -| `PYROSCOPE_URL` | Pyroscope 服务地址 | - | - -
- -
-多节点部署注意事项 - -- 所有节点必须使用相同的 `SESSION_SECRET`。 -- 使用共享 Redis 时必须使用相同的 `CRYPTO_SECRET`。 -- 每个节点应设置不同且稳定的 `NODE_NAME`。 -- 使用外部数据库、外部 Redis、统一 HTTPS 入口和可靠备份。 - -
- -
-从源码构建镜像 - -```bash -git clone https://github.com/MAX-API-Next/MAX-API.git -cd MAX-API - -go mod download -go mod verify -docker build --pull --no-cache -t cscitechtop/max-api:latest . -``` - -前端使用 Bun workspace。构建上下文需要保留 `web/package.json`、`web/bun.lock` 和 `web/default/package.json`，否则 `catalog:` 依赖可能无法解析。 - -
- -## 🗺️ 路线图 - -以下方向会根据维护节奏和真实场景调整，不构成发布日期承诺： - -- 深化模型目录、权限、能力标签、价格和供应商变化治理。 -- 完善 Agent 调用追踪、成本归因、异常定位、Evidence 与 AgentOps。 -- 以受控插件形式提供 Detector、OpsAgent、评估器和行业治理能力。 -- 增强多模态任务、协议转换、国产模型和供应商模板。 -- 完善组织级权限、审计、私有化、多节点和运营体验。 - -欢迎通过 [GitHub Issues](https://github.com/MAX-API-Next/MAX-API/issues) 提交需求和改进建议。 - -## ❓ 常见问题 - -
-MAX API 会提供模型服务或 API Key 吗？ - -不会。MAX API 是模型与 Agent 工作负载的网关治理层。你需要自行获得合法授权的上游模型服务和凭据。 - -
- -
-MAX API 和 Agent 框架是什么关系？ - -MAX API 不替代 Dify、LangChain、MCP Server、工作流引擎或业务 Agent。它负责这些应用与上游模型之间的访问、路由、成本、日志和治理。 - -
- -
-支持哪些数据库？ - -支持 SQLite、MySQL ≥ 5.7.8 和 PostgreSQL ≥ 9.6。SQLite 适合本地体验；生产建议使用 MySQL 或 PostgreSQL。 - -
- -
-能否从 One API / New API 迁移？ - -项目兼容 One API 与 New API 的主要数据结构，但迁移前仍应备份数据库，并在测试环境验证渠道、倍率、用户、令牌、任务和日志数据。 - -
- -
-图像、流式响应或大响应被截断怎么办？ - -可以调整 `STREAM_SCANNER_MAX_BUFFER_MB`。请求体返回 `413` 时检查 `MAX_REQUEST_BODY_MB`。提高限制前请同时评估内存与并发风险。 - -
- -
-多机部署登录状态不一致怎么办？ - -确认所有节点使用相同的 `SESSION_SECRET`。使用共享 Redis 时，还必须统一 `CRYPTO_SECRET`，并为节点设置稳定且不同的 `NODE_NAME`。 - -
- -
-普通用户能看到管理员日志审计内容吗？ - -普通用户日志接口会过滤管理员专用字段。数据库管理员、系统管理员或拥有管理员日志权限的人仍可能访问相关数据，因此必须严格控制权限和留存策略。 - -
- -## 🔗 资源与社区 - -| 资源 | 链接 | -|---|---| -| 项目仓库 | [MAX-API-Next/MAX-API](https://github.com/MAX-API-Next/MAX-API) | -| 问题反馈 | [GitHub Issues](https://github.com/MAX-API-Next/MAX-API/issues) | -| 最新发布 | [GitHub Releases](https://github.com/MAX-API-Next/MAX-API/releases) | -| Docker 镜像 | [cscitechtop/max-api](https://hub.docker.com/r/cscitechtop/max-api) | -| DeepWiki | [Ask DeepWiki](https://deepwiki.com/MAX-API-Next/MAX-API) | -| 社群交流 | QQ 群 `950126533`；微信群搜索 `MAX-API` | - -配套工具： - -- [max-api-key-tool](https://github.com/MAX-API-Next/MAX-API-key-tool)：Key 额度查询工具。 -- [max-api-horizon](https://github.com/MAX-API-Next/MAX-API-horizon)：MAX API 高性能优化版。 - -## 🤝 项目来源与二次开发 - -MAX API 基于以下开源项目持续增强 AI API 网关与治理能力、扩展功能并修复问题： - -| 项目 | 许可证 | -|---|---| -| [One API](https://github.com/songquanpeng/one-api) | MIT | -| [New API](https://github.com/QuantumNous/new-api) | AGPLv3 | -| [Midjourney-Proxy](https://github.com/novicezk/midjourney-proxy) | Apache-2.0 | -| [Suno API](https://github.com/Suno-API/Suno-API) | MIT | - -如果你基于本项目进行二次开发并仅供自用，欢迎在项目主页、页脚或“关于”页面等明显位置，任选一种方式保留项目来源或社区鸣谢： - -- 添加项目地址：[MAX-API-Next/MAX-API](https://github.com/MAX-API-Next/MAX-API) -- 鸣谢社区：[MAX-API-Next](https://github.com/MAX-API-Next) - -
-React / Tailwind CSS 鸣谢代码示例 - -```tsx -

- 基于{' '} - - MAX-API-Next/MAX-API - {' '} - 二次开发 · 感谢{' '} - - MAX-API-Next 社区 - -

-``` - -
- -满足上述任一展示要求并保持链接清晰可见，即自动获得本项目的临时商用授权，无需另行申请或等待确认。该授权为非永久授权，仅在持续满足展示要求期间有效；有效期及后续调整以项目方在本 README 或官方社区发布的最新说明为准。 - -商业使用本项目时，还须分别遵守 One API 的 MIT 许可证与 New API 的 AGPLv3 许可证，具体以各上游项目的 `LICENSE` 文件为准。本项目提供的临时商用授权不替代、也不免除相关上游项目的开源许可义务。 - -如不再满足展示要求，或临时授权到期、被公告调整或终止，仍需按 AGPLv3 或项目方另行书面授权使用本项目。如需长期商用授权，请联系：maxapi@max-api.ai。 - -## 📜 许可证 - -本项目采用 [GNU Affero 通用公共许可证 v3.0（AGPLv3）](./LICENSE) 授权。 - -除默认的 AGPLv3 授权外，符合上文“项目来源与二次开发”条件的自用二开项目，可按该说明自动获得非永久的临时商用授权。该临时授权仅覆盖 MAX API 项目方有权授权的新增与修改部分，并不包含或代替 One API、New API 等上游项目的许可授权。 - -如果你修改并通过网络向用户提供本项目服务，请理解并遵守 AGPLv3 对应源码提供等义务。商业合作、机构合作或其他授权问题，请联系：maxapi@max-api.ai。 - ---- - -
- -### 💖 感谢使用 MAX API - -如果这个项目对你有帮助，欢迎给我们一个 ⭐ Star。 - -**[官方文档](https://github.com/MAX-API-Next/MAX-API)** • **[问题反馈](https://github.com/MAX-API-Next/MAX-API/issues)** • **[最新发布](https://github.com/MAX-API-Next/MAX-API/releases)** - -Built with ❤️ by MAX-API-Next - -
diff --git a/controller/channel_performance.go b/controller/channel_performance.go index d48201d0..bff1a5d7 100644 --- a/controller/channel_performance.go +++ b/controller/channel_performance.go @@ -1,12 +1,10 @@ package controller import ( - "errors" "net/http" "strconv" "strings" - "github.com/MAX-API-Next/MAX-API/logger" "github.com/MAX-API-Next/MAX-API/service" "github.com/gin-gonic/gin" ) @@ -27,20 +25,7 @@ func GetChannelPerformance(c *gin.Context) { result, err := service.GetChannelPerformance(c.Request.Context(), query) if err != nil { - status := http.StatusInternalServerError - if errors.Is(err, service.ErrInvalidChannelPerformanceQuery) { - status = http.StatusBadRequest - } else { - logger.LogError(c.Request.Context(), "failed to query channel performance: "+err.Error()) - } - message := err.Error() - if status == http.StatusInternalServerError { - message = "failed to query channel performance" - } - c.JSON(status, gin.H{ - "success": false, - "message": message, - }) + respondPerformanceError(c, err, service.ErrInvalidChannelPerformanceQuery, "failed to query channel performance") return } @@ -58,20 +43,7 @@ func GetChannelPerformanceDetail(c *gin.Context) { parseIntQuery(c, "channel_id"), ) if err != nil { - status := http.StatusInternalServerError - if errors.Is(err, service.ErrInvalidChannelPerformanceQuery) { - status = http.StatusBadRequest - } else { - logger.LogError(c.Request.Context(), "failed to query channel performance detail: "+err.Error()) - } - message := err.Error() - if status == http.StatusInternalServerError { - message = "failed to query channel performance detail" - } - c.JSON(status, gin.H{ - "success": false, - "message": message, - }) + respondPerformanceError(c, err, service.ErrInvalidChannelPerformanceQuery, "failed to query channel performance detail") return } diff --git a/controller/model_performance.go b/controller/model_performance.go index 2a7f0ee2..51bfc330 100644 --- a/controller/model_performance.go +++ b/controller/model_performance.go @@ -1,11 +1,9 @@ package controller import ( - "errors" "net/http" "strings" - "github.com/MAX-API-Next/MAX-API/logger" "github.com/MAX-API-Next/MAX-API/service" "github.com/gin-gonic/gin" ) @@ -25,20 +23,7 @@ func GetModelPerformance(c *gin.Context) { result, err := service.GetModelPerformance(c.Request.Context(), query) if err != nil { - status := http.StatusInternalServerError - if errors.Is(err, service.ErrInvalidModelPerformanceQuery) { - status = http.StatusBadRequest - } else { - logger.LogError(c.Request.Context(), "failed to query model performance: "+err.Error()) - } - message := err.Error() - if status == http.StatusInternalServerError { - message = "failed to query model performance" - } - c.JSON(status, gin.H{ - "success": false, - "message": message, - }) + respondPerformanceError(c, err, service.ErrInvalidModelPerformanceQuery, "failed to query model performance") return } @@ -57,20 +42,7 @@ func GetModelPerformanceDetail(c *gin.Context) { strings.TrimSpace(c.Query("model")), ) if err != nil { - status := http.StatusInternalServerError - if errors.Is(err, service.ErrInvalidModelPerformanceQuery) { - status = http.StatusBadRequest - } else { - logger.LogError(c.Request.Context(), "failed to query model performance detail: "+err.Error()) - } - message := err.Error() - if status == http.StatusInternalServerError { - message = "failed to query model performance detail" - } - c.JSON(status, gin.H{ - "success": false, - "message": message, - }) + respondPerformanceError(c, err, service.ErrInvalidModelPerformanceQuery, "failed to query model performance detail") return } diff --git a/controller/performance_error.go b/controller/performance_error.go new file mode 100644 index 00000000..b6843f23 --- /dev/null +++ b/controller/performance_error.go @@ -0,0 +1,24 @@ +package controller + +import ( + "errors" + "net/http" + + "github.com/MAX-API-Next/MAX-API/logger" + "github.com/gin-gonic/gin" +) + +func respondPerformanceError(c *gin.Context, err error, invalidQueryError error, fallbackMessage string) { + status := http.StatusInternalServerError + message := fallbackMessage + if errors.Is(err, invalidQueryError) { + status = http.StatusBadRequest + message = err.Error() + } else { + logger.LogError(c.Request.Context(), fallbackMessage+": "+err.Error()) + } + c.JSON(status, gin.H{ + "success": false, + "message": message, + }) +} diff --git a/docker-compose.yml b/docker-compose.yml index e0c5a420..05abd152 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,7 +33,7 @@ services: - ERROR_LOG_ENABLED=true # 是否启用错误日志记录 (Whether to enable error log recording) - BATCH_UPDATE_ENABLED=true # 是否启用批量更新 (Whether to enable batch update) - NODE_NAME=max-api-node-1 # 节点名称，用于审计日志中标识节点身份；多节点/容器部署时建议设置 (Node name used in audit logs; recommended when running multiple instances or in containers) -# - STREAMING_TIMEOUT=300 # 流模式无响应超时时间，单位秒，默认120秒，如果出现空补全可以尝试改为更大值 （Streaming timeout in seconds, default is 120s. Increase if experiencing empty completions） +# - STREAMING_TIMEOUT=300 # 流模式无响应超时时间，单位秒，默认300秒，如果出现空补全可以尝试改为更大值 （Streaming timeout in seconds, default is 300s. Increase if experiencing empty completions） # - SESSION_SECRET=random_string # 多机部署时设置，必须修改这个随机字符串！！ （multi-node deployment, set this to a random string!!!!!!!） # - SYNC_FREQUENCY=60 # Uncomment if regular database syncing is needed # - GOOGLE_ANALYTICS_ID=G-XXXXXXXXXX # Google Analytics 的测量 ID (Google Analytics Measurement ID) diff --git a/model/perf_metric.go b/model/perf_metric.go index 9480a6cd..8c470983 100644 --- a/model/perf_metric.go +++ b/model/perf_metric.go @@ -1,6 +1,7 @@ package model import ( + "context" "time" "github.com/google/uuid" @@ -113,8 +114,12 @@ func CommitPerfMetricBucket(receiptKey string, metric *PerfMetric) (bool, error) } func GetPerfMetrics(modelName string, group string, startTs int64, endTs int64) ([]PerfMetric, error) { + return GetPerfMetricsContext(context.Background(), modelName, group, startTs, endTs) +} + +func GetPerfMetricsContext(ctx context.Context, modelName string, group string, startTs int64, endTs int64) ([]PerfMetric, error) { var metrics []PerfMetric - query := DB.Model(&PerfMetric{}). + query := DB.WithContext(ctx).Model(&PerfMetric{}). Where("model_name = ? AND bucket_ts >= ? AND bucket_ts <= ?", modelName, startTs, endTs) if group != "" { query = query.Where(map[string]interface{}{"group": group}) @@ -135,8 +140,12 @@ type PerfMetricSummary struct { } func GetPerfMetricsSummaryAll(startTs int64, endTs int64, groups []string) ([]PerfMetricSummary, error) { + return GetPerfMetricsSummaryAllContext(context.Background(), startTs, endTs, groups) +} + +func GetPerfMetricsSummaryAllContext(ctx context.Context, startTs int64, endTs int64, groups []string) ([]PerfMetricSummary, error) { var summaries []PerfMetricSummary - query := DB.Model(&PerfMetric{}). + query := DB.WithContext(ctx).Model(&PerfMetric{}). Select("model_name, SUM(request_count) as request_count, SUM(success_count) as success_count, SUM(total_latency_ms) as total_latency_ms, SUM(output_tokens) as output_tokens, SUM(generation_ms) as generation_ms, MIN(bucket_ts) as min_bucket_ts, MAX(bucket_ts) as max_bucket_ts"). Where("bucket_ts >= ? AND bucket_ts <= ?", startTs, endTs) if groups != nil { @@ -167,16 +176,27 @@ func DeletePerfMetricFlushReceiptsBefore(cutoffTs int64) error { } func GetPerfMetricFlushReceiptKeys(receiptKeys []string) (map[string]struct{}, error) { + return GetPerfMetricFlushReceiptKeysContext(context.Background(), receiptKeys) +} + +func GetPerfMetricFlushReceiptKeysContext(ctx context.Context, receiptKeys []string) (map[string]struct{}, error) { result := make(map[string]struct{}, len(receiptKeys)) if len(receiptKeys) == 0 { return result, nil } - var receipts []PerfMetricFlushReceipt - if err := DB.Select("receipt_key").Where("receipt_key IN ?", receiptKeys).Find(&receipts).Error; err != nil { - return nil, err - } - for _, receipt := range receipts { - result[receipt.ReceiptKey] = struct{}{} + const batchSize = 200 + for start := 0; start < len(receiptKeys); start += batchSize { + end := min(start+batchSize, len(receiptKeys)) + var receipts []PerfMetricFlushReceipt + if err := DB.WithContext(ctx). + Select("receipt_key"). + Where("receipt_key IN ?", receiptKeys[start:end]). + Find(&receipts).Error; err != nil { + return nil, err + } + for _, receipt := range receipts { + result[receipt.ReceiptKey] = struct{}{} + } } return result, nil } diff --git a/model/perf_metric_test.go b/model/perf_metric_test.go new file mode 100644 index 00000000..911930f8 --- /dev/null +++ b/model/perf_metric_test.go @@ -0,0 +1,43 @@ +package model + +import ( + "context" + "fmt" + "testing" + + "github.com/glebarez/sqlite" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +func TestGetPerfMetricFlushReceiptKeysBatchesLargeLookups(t *testing.T) { + previousDB := DB + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&PerfMetricFlushReceipt{})) + DB = db + t.Cleanup(func() { DB = previousDB }) + + receipts := make([]PerfMetricFlushReceipt, 0, 7) + requested := make([]string, 0, 1_205) + for index := 0; index < 1_205; index++ { + key := fmt.Sprintf("receipt-%04d", index) + requested = append(requested, key) + if index%200 == 0 { + receipts = append(receipts, PerfMetricFlushReceipt{ + ReceiptKey: key, + ClaimToken: fmt.Sprintf("claim-%04d", index), + CreatedAt: int64(index + 1), + }) + } + } + require.NoError(t, db.Create(&receipts).Error) + + result, err := GetPerfMetricFlushReceiptKeysContext(context.Background(), requested) + + require.NoError(t, err) + require.Len(t, result, len(receipts)) + for _, receipt := range receipts { + require.Contains(t, result, receipt.ReceiptKey) + } +} diff --git a/pkg/perf_metrics/flush.go b/pkg/perf_metrics/flush.go index 9de14abf..7b31ff95 100644 --- a/pkg/perf_metrics/flush.go +++ b/pkg/perf_metrics/flush.go @@ -138,7 +138,7 @@ func deleteRedisBucket(key bucketKey) error { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() redisKey := redisBucketKey(key) - if err := common.RDB.HSet(ctx, redisKey, "flushed", 1).Err(); err != nil { + if err := markRedisBucketFlushed(ctx, redisKey); err != nil { return err } pipe := common.RDB.TxPipeline() @@ -150,6 +150,14 @@ func deleteRedisBucket(key bucketKey) error { return err } +func markRedisBucketFlushed(ctx context.Context, redisKey string) error { + pipe := common.RDB.TxPipeline() + pipe.HSet(ctx, redisKey, "flushed", 1) + pipe.Expire(ctx, redisKey, redisBucketTTL) + _, err := pipe.Exec(ctx) + return err +} + func cleanupExpiredMetrics(retentionDays int) { receiptCutoff := time.Now().Add(-redisBucketTTL - time.Hour).Unix() if err := model.DeletePerfMetricFlushReceiptsBefore(receiptCutoff); err != nil { diff --git a/pkg/perf_metrics/metrics.go b/pkg/perf_metrics/metrics.go index 789fe86f..dacf6302 100644 --- a/pkg/perf_metrics/metrics.go +++ b/pkg/perf_metrics/metrics.go @@ -16,6 +16,7 @@ import ( relaycommon "github.com/MAX-API-Next/MAX-API/relay/common" "github.com/MAX-API-Next/MAX-API/setting/perf_metrics_setting" "github.com/go-redis/redis/v8" + "github.com/google/uuid" ) var hotBuckets sync.Map @@ -33,6 +34,11 @@ var committedBuckets sync.Map var projectionGapBuckets sync.Map var projectionGapCount atomic.Int64 var projectionDegradedUntil atomic.Int64 +var restoredProjectionDegraded atomic.Bool +var projectionNodeClaimConflict atomic.Bool +var projectionNodeClaimToken = uuid.NewString() + +var getPerfMetricFlushReceiptKeys = model.GetPerfMetricFlushReceiptKeysContext const ( redisActiveBucketIndex = "perf:v2:active" @@ -40,6 +46,7 @@ const ( redisBucketTTL = 48 * time.Hour projectionHealthInterval = 5 * time.Second projectionHealthyTTL = 30 * time.Second + projectionNodeClaimTTL = 30 * time.Second ) var redisBucketSnapshotScript = redis.NewScript(` @@ -67,21 +74,40 @@ redis.call('EXPIRE', KEYS[4], tonumber(ARGV[15])) return 1 `) +var redisNodeClaimScript = redis.NewScript(` +local owner = redis.call('GET', KEYS[1]) +if not owner then + redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2], 'NX') + owner = redis.call('GET', KEYS[1]) +end +if owner == ARGV[1] then + redis.call('EXPIRE', KEYS[1], ARGV[2]) + return 1 +end +return 0 +`) + // seriesSchema is a stable client cache/schema marker. Do not change it when // hiding fields or making response-only privacy hardening changes. const seriesSchema = "dbcd0a3c01b55203" func Init() { - if common.RedisEnabled && common.RDB != nil && currentNodeName() == "unnamed" { - common.SysError("performance metrics Redis recovery requires a unique stable NODE_NAME per running instance") + if common.RedisEnabled && common.RDB != nil { + if currentNodeName() == "unnamed" { + common.SysError("performance metrics Redis recovery requires a unique stable NODE_NAME per running instance") + } + maintainProjectionNodeClaim() + } + if err := restoreProjectionHealth(); err != nil { + common.SysError("failed to restore performance projection health: " + err.Error()) } if recovered, err := recoverRedisBucketsForNode(); err != nil { common.SysError("failed to recover active performance metric buckets from Redis: " + err.Error()) } else if recovered > 0 { common.SysLog(fmt.Sprintf("recovered %d active performance metric buckets from Redis", recovered)) } - if err := restoreProjectionHealth(); err != nil { - common.SysError("failed to restore performance projection health: " + err.Error()) + if err := publishProjectionHealth(); err != nil && common.RedisEnabled { + common.SysError("failed to publish initial performance projection health: " + err.Error()) } go flushLoop() go projectionHealthLoop() @@ -163,7 +189,11 @@ func Record(sample Sample) { } func Query(params QueryParams) (QueryResult, error) { - result, err := QueryDetailed(params) + return QueryContext(context.Background(), params) +} + +func QueryContext(ctx context.Context, params QueryParams) (QueryResult, error) { + result, err := QueryDetailedContext(ctx, params) if err != nil { return QueryResult{}, err } @@ -171,6 +201,10 @@ func Query(params QueryParams) (QueryResult, error) { } func QueryDetailed(params QueryParams) (DetailedQueryResult, error) { + return QueryDetailedContext(context.Background(), params) +} + +func QueryDetailedContext(ctx context.Context, params QueryParams) (DetailedQueryResult, error) { setting := perf_metrics_setting.GetSetting() bucketSeconds := perf_metrics_setting.GetBucketSeconds() if params.Hours <= 0 { @@ -184,7 +218,7 @@ func QueryDetailed(params QueryParams) (DetailedQueryResult, error) { merged := map[bucketKey]counters{} metricStartTs := bucketStartFor(startTs, bucketSeconds) - rows, err := model.GetPerfMetrics(params.Model, params.Group, metricStartTs, endTs) + rows, err := model.GetPerfMetricsContext(ctx, params.Model, params.Group, metricStartTs, endTs) if err != nil { return DetailedQueryResult{}, err } @@ -208,7 +242,7 @@ func QueryDetailed(params QueryParams) (DetailedQueryResult, error) { }) } - activeState := mergeActiveBuckets(merged, activeBucketFilter{ + activeState := mergeActiveBuckets(ctx, merged, activeBucketFilter{ model: params.Model, group: params.Group, allowedGroups: allowedGroupSet(params.AllowedGroups), @@ -241,7 +275,7 @@ func QuerySummaryAll(hours int, groups []string) (SummaryAllResult, error) { // an operational client also needs the requested range, selected bucket // coverage, bucket size, and collection state. func QuerySummaryAllRange(startTs int64, endTs int64, groups []string) (SummaryAllResult, error) { - result, err := QuerySummaryAllRangeDetailed(startTs, endTs, groups) + result, err := QuerySummaryAllRangeDetailedContext(context.Background(), startTs, endTs, groups) if err != nil { return SummaryAllResult{}, err } @@ -249,6 +283,10 @@ func QuerySummaryAllRange(startTs int64, endTs int64, groups []string) (SummaryA } func QuerySummaryAllRangeDetailed(startTs int64, endTs int64, groups []string) (DetailedSummaryAllResult, error) { + return QuerySummaryAllRangeDetailedContext(context.Background(), startTs, endTs, groups) +} + +func QuerySummaryAllRangeDetailedContext(ctx context.Context, startTs int64, endTs int64, groups []string) (DetailedSummaryAllResult, error) { setting := perf_metrics_setting.GetSetting() bucketSeconds := perf_metrics_setting.GetBucketSeconds() if endTs <= startTs { @@ -265,7 +303,7 @@ func QuerySummaryAllRangeDetailed(startTs int64, endTs int64, groups []string) ( allowedGroups := allowedGroupSet(groups) metricStartTs := bucketStartFor(startTs, bucketSeconds) - rows, err := model.GetPerfMetricsSummaryAll(metricStartTs, endTs, groups) + rows, err := model.GetPerfMetricsSummaryAllContext(ctx, metricStartTs, endTs, groups) if err != nil { return DetailedSummaryAllResult{}, err } @@ -285,7 +323,7 @@ func QuerySummaryAllRangeDetailed(startTs int64, endTs int64, groups []string) ( } active := map[bucketKey]counters{} - activeState := mergeActiveBuckets(active, activeBucketFilter{ + activeState := mergeActiveBuckets(ctx, active, activeBucketFilter{ allowedGroups: allowedGroups, startTs: metricStartTs, endTs: endTs, @@ -620,13 +658,13 @@ type activeBucketFilter struct { endTs int64 } -func mergeActiveBuckets(merged map[bucketKey]counters, filter activeBucketFilter) activeCollectionHealth { +func mergeActiveBuckets(ctx context.Context, merged map[bucketKey]counters, filter activeBucketFilter) activeCollectionHealth { if !common.RedisEnabled || common.RDB == nil { mergeMemoryBuckets(merged, &hotBuckets, filter, false) return activeCollectionHealthy } - redisBuckets, err := loadRedisActiveBuckets(filter) + redisBuckets, err := loadRedisActiveBuckets(ctx, filter) if err != nil { mergeMemoryBuckets(merged, &hotBuckets, filter, false) return activeCollectionFailed @@ -641,7 +679,7 @@ func mergeActiveBuckets(merged map[bucketKey]counters, filter activeBucketFilter if localProjectionDegraded() { return activeCollectionFailed } - degraded, err := loadProjectionHealth() + degraded, err := loadProjectionHealth(ctx) if err != nil || degraded { return activeCollectionFailed } @@ -732,8 +770,8 @@ func recordRedis(key bucketKey, value counters) bool { return true } -func loadRedisActiveBuckets(filter activeBucketFilter) (map[bucketKey]counters, error) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) +func loadRedisActiveBuckets(parent context.Context, filter activeBucketFilter) (map[bucketKey]counters, error) { + ctx, cancel := context.WithTimeout(parent, time.Second) defer cancel() index := redisActiveBucketIndex if filter.model != "" { @@ -748,11 +786,13 @@ func loadRedisActiveBuckets(filter activeBucketFilter) (map[bucketKey]counters, return nil, err } type decodedBucket struct { - key bucketKey - value counters + key bucketKey + value counters + receiptKey string } decoded := make([]decodedBucket, 0, len(values)) receiptKeys := make([]string, 0, len(values)) + now := time.Now().Unix() for _, fields := range values { key, value, ok := decodeRedisBucket(fields) if !ok || !matchesActiveBucket(key, filter) { @@ -761,17 +801,27 @@ func loadRedisActiveBuckets(filter activeBucketFilter) (map[bucketKey]counters, if _, suppressed := committedBuckets.Load(key); suppressed { continue } - decoded = append(decoded, decodedBucket{key: key, value: value}) - receiptKeys = append(receiptKeys, perfMetricReceiptKey(key)) - } - receipts, err := model.GetPerfMetricFlushReceiptKeys(receiptKeys) - if err != nil { - return nil, err + bucket := decodedBucket{key: key, value: value} + if key.bucketSeconds > 0 && key.bucketTs+key.bucketSeconds <= now { + bucket.receiptKey = perfMetricReceiptKey(key) + receiptKeys = append(receiptKeys, bucket.receiptKey) + } + decoded = append(decoded, bucket) + } + receipts := map[string]struct{}{} + if len(receiptKeys) > 0 { + var err error + receipts, err = getPerfMetricFlushReceiptKeys(ctx, receiptKeys) + if err != nil { + return nil, err + } } merged := make(map[bucketKey]counters, len(decoded)) for _, bucket := range decoded { - if _, committed := receipts[perfMetricReceiptKey(bucket.key)]; committed { - continue + if bucket.receiptKey != "" { + if _, committed := receipts[bucket.receiptKey]; committed { + continue + } } mergeCounters(merged, bucket.key, bucket.value) } @@ -804,6 +854,11 @@ func redisProjectionHealthKey(node string) string { return fmt.Sprintf("perf:v2:health:%x", identity[:12]) } +func redisProjectionNodeClaimKey(node string) string { + identity := sha256.Sum256([]byte(node)) + return fmt.Sprintf("perf:v2:node-claim:%x", identity[:12]) +} + func setProjectionGap(key bucketKey, incomplete bool) { if incomplete { if _, loaded := projectionGapBuckets.LoadOrStore(key, struct{}{}); !loaded { @@ -834,15 +889,58 @@ func localProjectionDegraded() bool { } func projectionHealthLoop() { + if !common.RedisEnabled || common.RDB == nil { + return + } ticker := time.NewTicker(projectionHealthInterval) defer ticker.Stop() for range ticker.C { + maintainProjectionNodeClaim() if err := publishProjectionHealth(); err != nil && common.RedisEnabled { common.SysError("failed to publish performance projection health: " + err.Error()) } } } +func maintainProjectionNodeClaim() { + if !common.RedisEnabled || common.RDB == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + owned, err := claimProjectionNode(ctx, currentNodeName(), projectionNodeClaimToken) + if err != nil { + common.SysError("failed to refresh performance metrics NODE_NAME claim: " + err.Error()) + return + } + if !owned { + if projectionNodeClaimConflict.CompareAndSwap(false, true) { + common.SysError(fmt.Sprintf("duplicate NODE_NAME %q detected for performance metrics; use a unique stable NODE_NAME per running instance", currentNodeName())) + } + return + } + if projectionNodeClaimConflict.CompareAndSwap(true, false) { + common.SysLog(fmt.Sprintf("performance metrics NODE_NAME claim recovered for %q", currentNodeName())) + } +} + +func claimProjectionNode(ctx context.Context, node string, token string) (bool, error) { + if node == "" || token == "" { + return false, fmt.Errorf("node name and claim token are required") + } + result, err := redisNodeClaimScript.Run( + ctx, + common.RDB, + []string{redisProjectionNodeClaimKey(node)}, + token, + int64(projectionNodeClaimTTL/time.Second), + ).Int() + if err != nil { + return false, err + } + return result == 1, nil +} + func publishProjectionHealth() error { if !common.RedisEnabled || common.RDB == nil { return nil @@ -883,6 +981,7 @@ func restoreProjectionHealth() error { degradedUntil := parseRedisInt(fields["degraded_until"]) if fields["degraded"] == "1" && degradedUntil > time.Now().Unix() { projectionDegradedUntil.Store(degradedUntil) + restoredProjectionDegraded.Store(true) } return nil } @@ -891,13 +990,15 @@ func resetProjectionHealthForTest() { projectionGapBuckets = sync.Map{} projectionGapCount.Store(0) projectionDegradedUntil.Store(0) + restoredProjectionDegraded.Store(false) + projectionNodeClaimConflict.Store(false) } -func loadProjectionHealth() (bool, error) { +func loadProjectionHealth(parent context.Context) (bool, error) { if !common.RedisEnabled || common.RDB == nil { return false, nil } - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(parent, time.Second) defer cancel() keys, err := activeRedisKeys(ctx, redisProjectionHealthIndex) if err != nil { @@ -907,8 +1008,9 @@ func loadProjectionHealth() (bool, error) { if err != nil { return false, err } + now := time.Now().Unix() for _, fields := range values { - if fields["degraded"] == "1" { + if fields["degraded"] == "1" && parseRedisInt(fields["degraded_until"]) > now { return true, nil } } @@ -1006,6 +1108,7 @@ func recoverRedisBucketsForNode() (int, error) { if err != nil { return 0, err } + restoreGapOwnership := restoredProjectionDegraded.Swap(false) recovered := 0 for _, candidate := range candidates { if _, committed := receipts[perfMetricReceiptKey(candidate.key)]; committed { @@ -1015,9 +1118,18 @@ func recoverRedisBucketsForNode() (int, error) { continue } bucket := &atomicBucket{} - bucket.addCounters(candidate.value) + if !bucket.addCounters(candidate.value) { + common.SysError(fmt.Sprintf("failed to restore performance metric bucket model=%s group=%s bucket=%d", candidate.key.model, candidate.key.group, candidate.key.bucketTs)) + continue + } hotBuckets.Store(candidate.key, bucket) + if restoreGapOwnership { + setProjectionGap(candidate.key, true) + } recovered++ } + if restoreGapOwnership && recovered == 0 { + projectionDegradedUntil.Store(0) + } return recovered, nil } diff --git a/pkg/perf_metrics/metrics_test.go b/pkg/perf_metrics/metrics_test.go index eecc6e01..1bf17ef4 100644 --- a/pkg/perf_metrics/metrics_test.go +++ b/pkg/perf_metrics/metrics_test.go @@ -19,12 +19,18 @@ import ( func TestQuerySummaryAllRangeReportsBucketCoverageAndGroupFilter(t *testing.T) { previousDB := model.DB + previousRDB := common.RDB + previousRedisEnabled := common.RedisEnabled db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) require.NoError(t, err) model.DB = db + common.RDB = nil + common.RedisEnabled = false hotBuckets = sync.Map{} t.Cleanup(func() { model.DB = previousDB + common.RDB = previousRDB + common.RedisEnabled = previousRedisEnabled hotBuckets = sync.Map{} }) require.NoError(t, db.AutoMigrate(&model.PerfMetric{}, &model.PerfMetricFlushReceipt{})) @@ -334,12 +340,18 @@ func TestQueryDetailedMarksRedisFailureAsQueryFailedWithoutSamples(t *testing.T) func TestQuerySummaryAllRangeDetailedMarksEmptyWindowAsNoSamples(t *testing.T) { previousDB := model.DB + previousRDB := common.RDB + previousRedisEnabled := common.RedisEnabled db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) require.NoError(t, err) model.DB = db + common.RDB = nil + common.RedisEnabled = false hotBuckets = sync.Map{} t.Cleanup(func() { model.DB = previousDB + common.RDB = previousRDB + common.RedisEnabled = previousRedisEnabled hotBuckets = sync.Map{} }) require.NoError(t, db.AutoMigrate(&model.PerfMetric{}, &model.PerfMetricFlushReceipt{})) @@ -652,7 +664,7 @@ func TestModelQueryUsesModelScopedRedisIndex(t *testing.T) { require.True(t, recordRedis(betaKey, counters{requestCount: 1, successCount: 1})) require.NoError(t, redisClient.Del(context.Background(), redisActiveBucketIndex).Err()) - values, err := loadRedisActiveBuckets(activeBucketFilter{ + values, err := loadRedisActiveBuckets(context.Background(), activeBucketFilter{ model: "alpha", startTs: alphaKey.bucketTs, endTs: now, @@ -662,6 +674,184 @@ func TestModelQueryUsesModelScopedRedisIndex(t *testing.T) { require.Contains(t, values, alphaKey) } +func TestLoadRedisActiveBucketsSkipsReceiptLookupForOpenBuckets(t *testing.T) { + previousRDB := common.RDB + previousRedisEnabled := common.RedisEnabled + previousLookup := getPerfMetricFlushReceiptKeys + redisServer := miniredis.RunT(t) + redisClient := redis.NewClient(&redis.Options{Addr: redisServer.Addr()}) + common.RDB = redisClient + common.RedisEnabled = true + lookupCalls := 0 + getPerfMetricFlushReceiptKeys = func(context.Context, []string) (map[string]struct{}, error) { + lookupCalls++ + return map[string]struct{}{}, nil + } + t.Cleanup(func() { + common.RDB = previousRDB + common.RedisEnabled = previousRedisEnabled + getPerfMetricFlushReceiptKeys = previousLookup + _ = redisClient.Close() + }) + + now := time.Now().Unix() + key := bucketKey{ + model: "alpha", + group: "prod", + node: "node-a", + bucketTs: bucketStartFor(now, 3_600), + bucketSeconds: 3_600, + } + require.True(t, recordRedis(key, counters{requestCount: 1, successCount: 1})) + + values, err := loadRedisActiveBuckets(context.Background(), activeBucketFilter{ + model: "alpha", + startTs: key.bucketTs, + endTs: now, + }) + + require.NoError(t, err) + require.Contains(t, values, key) + require.Zero(t, lookupCalls) +} + +func TestClaimProjectionNodeDetectsDuplicateOwnerAndRefreshesTTL(t *testing.T) { + previousRDB := common.RDB + previousRedisEnabled := common.RedisEnabled + redisServer := miniredis.RunT(t) + redisClient := redis.NewClient(&redis.Options{Addr: redisServer.Addr()}) + common.RDB = redisClient + common.RedisEnabled = true + t.Cleanup(func() { + common.RDB = previousRDB + common.RedisEnabled = previousRedisEnabled + _ = redisClient.Close() + }) + + ctx := context.Background() + owned, err := claimProjectionNode(ctx, "shared-node", "process-a") + require.NoError(t, err) + require.True(t, owned) + owned, err = claimProjectionNode(ctx, "shared-node", "process-b") + require.NoError(t, err) + require.False(t, owned) + owned, err = claimProjectionNode(ctx, "shared-node", "process-a") + require.NoError(t, err) + require.True(t, owned) + require.Greater(t, redisClient.TTL(ctx, redisProjectionNodeClaimKey("shared-node")).Val(), time.Duration(0)) +} + +func TestMarkRedisBucketFlushedPreservesExpiryWhenHashIsMissing(t *testing.T) { + previousRDB := common.RDB + previousRedisEnabled := common.RedisEnabled + redisServer := miniredis.RunT(t) + redisClient := redis.NewClient(&redis.Options{Addr: redisServer.Addr()}) + common.RDB = redisClient + common.RedisEnabled = true + t.Cleanup(func() { + common.RDB = previousRDB + common.RedisEnabled = previousRedisEnabled + _ = redisClient.Close() + }) + + ctx := context.Background() + require.NoError(t, markRedisBucketFlushed(ctx, "perf:v2:bucket:missing")) + require.Equal(t, "1", redisClient.HGet(ctx, "perf:v2:bucket:missing", "flushed").Val()) + require.Greater(t, redisClient.TTL(ctx, "perf:v2:bucket:missing").Val(), time.Duration(0)) +} + +func TestRecoveredProjectionHealthClearsWhenNoBucketCanOwnTheGap(t *testing.T) { + previousDB := model.DB + previousRDB := common.RDB + previousRedisEnabled := common.RedisEnabled + previousNodeName := common.NodeName + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&model.PerfMetric{}, &model.PerfMetricFlushReceipt{})) + redisServer := miniredis.RunT(t) + redisClient := redis.NewClient(&redis.Options{Addr: redisServer.Addr()}) + model.DB = db + common.RDB = redisClient + common.RedisEnabled = true + common.NodeName = "restored-health-node" + resetProjectionHealthForTest() + t.Cleanup(func() { + model.DB = previousDB + common.RDB = previousRDB + common.RedisEnabled = previousRedisEnabled + common.NodeName = previousNodeName + resetProjectionHealthForTest() + _ = redisClient.Close() + }) + + ctx := context.Background() + require.NoError(t, redisClient.HSet(ctx, redisProjectionHealthKey(common.NodeName), map[string]interface{}{ + "degraded": 1, + "degraded_until": time.Now().Add(time.Hour).Unix(), + }).Err()) + require.NoError(t, restoreProjectionHealth()) + require.True(t, localProjectionDegraded()) + + recovered, err := recoverRedisBucketsForNode() + + require.NoError(t, err) + require.Zero(t, recovered) + require.False(t, localProjectionDegraded()) +} + +func TestLoadProjectionHealthIgnoresExpiredDegradedMarker(t *testing.T) { + previousRDB := common.RDB + previousRedisEnabled := common.RedisEnabled + redisServer := miniredis.RunT(t) + redisClient := redis.NewClient(&redis.Options{Addr: redisServer.Addr()}) + common.RDB = redisClient + common.RedisEnabled = true + t.Cleanup(func() { + common.RDB = previousRDB + common.RedisEnabled = previousRedisEnabled + _ = redisClient.Close() + }) + + ctx := context.Background() + healthKey := redisProjectionHealthKey("stale-node") + require.NoError(t, redisClient.HSet(ctx, healthKey, map[string]interface{}{ + "degraded": 1, + "degraded_until": time.Now().Add(-time.Minute).Unix(), + }).Err()) + require.NoError(t, redisClient.ZAdd(ctx, redisProjectionHealthIndex, &redis.Z{ + Score: float64(time.Now().Add(time.Hour).Unix()), + Member: healthKey, + }).Err()) + + degraded, err := loadProjectionHealth(ctx) + + require.NoError(t, err) + require.False(t, degraded) +} + +func TestQueryDetailedContextHonorsRequestCancellation(t *testing.T) { + previousDB := model.DB + previousRDB := common.RDB + previousRedisEnabled := common.RedisEnabled + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&model.PerfMetric{}, &model.PerfMetricFlushReceipt{})) + model.DB = db + common.RDB = nil + common.RedisEnabled = false + t.Cleanup(func() { + model.DB = previousDB + common.RDB = previousRDB + common.RedisEnabled = previousRedisEnabled + }) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = QueryDetailedContext(ctx, QueryParams{Model: "alpha", Hours: 1}) + + require.ErrorIs(t, err, context.Canceled) +} + func TestBuildCoverageClampsKnownBucketEndToRequestedEnd(t *testing.T) { coverage := buildCoverage(2_400, 2_500, map[bucketKey]counters{ {model: "alpha", group: "prod", bucketTs: 2_480, bucketSeconds: 60}: { diff --git a/service/channel_performance.go b/service/channel_performance.go index 2ddd96a8..586ad03d 100644 --- a/service/channel_performance.go +++ b/service/channel_performance.go @@ -14,11 +14,7 @@ import ( var ErrInvalidChannelPerformanceQuery = errors.New("invalid channel performance query") const ( - defaultChannelPerformanceHours = 1 - channelPerformanceDetailHours = 24 - maxChannelPerformanceHours = 168 - defaultChannelPerformanceLimit = 100 - maxChannelPerformanceLimit = 200 + channelPerformanceDetailHours = 24 ) type ChannelPerformanceQuery = model.ChannelPerformanceQuery @@ -266,42 +262,25 @@ func GetChannelPerformanceDetail(ctx context.Context, channelID int) (ChannelPer return defaultChannelPerformanceReader.Summary(ctx, ChannelPerformanceQuery{ ChannelID: channelID, Hours: channelPerformanceDetailHours, - Limit: maxChannelPerformanceLimit, + Limit: maxPerformanceLimit, }) } func normalizeChannelPerformanceQuery(query ChannelPerformanceQuery) (ChannelPerformanceQuery, bool, error) { - now := time.Now().Unix() - rangeWasClamped := false - if query.EndAt <= 0 { - query.EndAt = now - } - if query.StartAt <= 0 { - hours := query.Hours - if hours <= 0 { - hours = defaultChannelPerformanceHours - } - if hours > maxChannelPerformanceHours { - hours = maxChannelPerformanceHours - rangeWasClamped = true - } - query.Hours = hours - query.StartAt = query.EndAt - int64(hours)*int64(time.Hour/time.Second) - } - if query.EndAt <= query.StartAt { - return query, false, fmt.Errorf("%w: end must be greater than start", ErrInvalidChannelPerformanceQuery) - } - if query.Limit <= 0 { - query.Limit = defaultChannelPerformanceLimit - } - if query.Limit > maxChannelPerformanceLimit { - query.Limit = maxChannelPerformanceLimit - } - maxWindow := int64(maxChannelPerformanceHours) * int64(time.Hour/time.Second) - if query.EndAt-query.StartAt > maxWindow { - query.StartAt = query.EndAt - maxWindow - rangeWasClamped = true + normalized, rangeWasClamped, err := normalizePerformanceQuery( + query.StartAt, + query.EndAt, + query.Hours, + query.Limit, + ErrInvalidChannelPerformanceQuery, + ) + if err != nil { + return query, false, err } + query.StartAt = normalized.StartAt + query.EndAt = normalized.EndAt + query.Hours = normalized.Hours + query.Limit = normalized.Limit return query, rangeWasClamped, nil } diff --git a/service/channel_performance_test.go b/service/channel_performance_test.go index b9d66071..dae2034f 100644 --- a/service/channel_performance_test.go +++ b/service/channel_performance_test.go @@ -29,8 +29,8 @@ func TestNormalizeChannelPerformanceQueryBoundsWindowAndLimit(t *testing.T) { require.NoError(t, err) require.True(t, clamped) - require.Equal(t, maxChannelPerformanceLimit, query.Limit) - require.EqualValues(t, maxChannelPerformanceHours, (query.EndAt-query.StartAt)/int64(time.Hour/time.Second)) + require.Equal(t, maxPerformanceLimit, query.Limit) + require.EqualValues(t, maxPerformanceHours, (query.EndAt-query.StartAt)/int64(time.Hour/time.Second)) } func TestNormalizeChannelPerformanceQueryUsesServerOwnedHourWindow(t *testing.T) { @@ -52,7 +52,7 @@ func TestNormalizeChannelPerformanceQueryUsesServerOwnedHourWindow(t *testing.T) query, clamped, err = normalizeChannelPerformanceQuery(ChannelPerformanceQuery{Hours: 999}) require.NoError(t, err) require.True(t, clamped) - require.EqualValues(t, maxChannelPerformanceHours*time.Hour/time.Second, query.EndAt-query.StartAt) + require.EqualValues(t, maxPerformanceHours*time.Hour/time.Second, query.EndAt-query.StartAt) } func TestNormalizeChannelPerformanceQueryRejectsReversedRange(t *testing.T) { @@ -76,7 +76,7 @@ func TestGetChannelPerformanceDetailUsesFixedChannelWindow(t *testing.T) { require.NoError(t, err) require.Equal(t, 17, reader.query.ChannelID) require.Equal(t, channelPerformanceDetailHours, reader.query.Hours) - require.Equal(t, maxChannelPerformanceLimit, reader.query.Limit) + require.Equal(t, maxPerformanceLimit, reader.query.Limit) require.Equal(t, channelPerformanceDetailHours, result.TimeRange.Hours) } diff --git a/service/model_performance.go b/service/model_performance.go index 2fa7abe3..3afae5c5 100644 --- a/service/model_performance.go +++ b/service/model_performance.go @@ -9,6 +9,7 @@ import ( "github.com/MAX-API-Next/MAX-API/common" "github.com/MAX-API-Next/MAX-API/constant" + "github.com/MAX-API-Next/MAX-API/logger" "github.com/MAX-API-Next/MAX-API/model" perfmetrics "github.com/MAX-API-Next/MAX-API/pkg/perf_metrics" ) @@ -100,7 +101,7 @@ func (legacyModelPerformanceReader) Summary(ctx context.Context, query ModelPerf if truncated { rows = rows[:query.Limit] } - throughput := queryModelPerformanceThroughput(query) + throughput := queryModelPerformanceThroughput(ctx, query) return buildLegacyModelPerformanceSummary( query, @@ -237,15 +238,16 @@ func buildLegacyModelPerformanceSummary( } } -var queryModelPerformanceSummaryRange = perfmetrics.QuerySummaryAllRangeDetailed +var queryModelPerformanceSummaryRange = perfmetrics.QuerySummaryAllRangeDetailedContext -func queryModelPerformanceThroughput(query ModelPerformanceQuery) modelPerformanceThroughputSnapshot { +func queryModelPerformanceThroughput(ctx context.Context, query ModelPerformanceQuery) modelPerformanceThroughputSnapshot { groups := []string(nil) if query.Group != "" { groups = []string{query.Group} } - result, err := queryModelPerformanceSummaryRange(query.StartAt, query.EndAt, groups) + result, err := queryModelPerformanceSummaryRange(ctx, query.StartAt, query.EndAt, groups) if err != nil { + logger.LogError(ctx, "failed to query model performance throughput: "+err.Error()) return modelPerformanceThroughputSnapshot{State: perfmetrics.CollectionStateQueryFailed} } byModel := make(map[string]float64, len(result.Models)) @@ -265,48 +267,31 @@ func GetModelPerformance(ctx context.Context, query ModelPerformanceQuery) (Mode return defaultModelPerformanceReader.Summary(ctx, query) } -func GetModelPerformanceDetail(_ context.Context, modelName string) (perfmetrics.DetailedQueryResult, error) { +func GetModelPerformanceDetail(ctx context.Context, modelName string) (perfmetrics.DetailedQueryResult, error) { modelName = strings.TrimSpace(modelName) if modelName == "" { return perfmetrics.DetailedQueryResult{}, fmt.Errorf("%w: model is required", ErrInvalidModelPerformanceQuery) } - return perfmetrics.QueryDetailed(perfmetrics.QueryParams{ + return perfmetrics.QueryDetailedContext(ctx, perfmetrics.QueryParams{ Model: modelName, Hours: modelPerformanceDetailHours, }) } func normalizeModelPerformanceQuery(query ModelPerformanceQuery) (ModelPerformanceQuery, bool, error) { - now := time.Now().Unix() - rangeWasClamped := false - if query.EndAt <= 0 { - query.EndAt = now - } - if query.StartAt <= 0 { - hours := query.Hours - if hours <= 0 { - hours = defaultChannelPerformanceHours - } - if hours > maxChannelPerformanceHours { - hours = maxChannelPerformanceHours - rangeWasClamped = true - } - query.Hours = hours - query.StartAt = query.EndAt - int64(hours)*int64(time.Hour/time.Second) - } - if query.EndAt <= query.StartAt { - return query, false, fmt.Errorf("%w: end must be greater than start", ErrInvalidModelPerformanceQuery) - } - if query.Limit <= 0 { - query.Limit = defaultChannelPerformanceLimit - } - if query.Limit > maxChannelPerformanceLimit { - query.Limit = maxChannelPerformanceLimit - } - maxWindow := int64(maxChannelPerformanceHours) * int64(time.Hour/time.Second) - if query.EndAt-query.StartAt > maxWindow { - query.StartAt = query.EndAt - maxWindow - rangeWasClamped = true + normalized, rangeWasClamped, err := normalizePerformanceQuery( + query.StartAt, + query.EndAt, + query.Hours, + query.Limit, + ErrInvalidModelPerformanceQuery, + ) + if err != nil { + return query, false, err } + query.StartAt = normalized.StartAt + query.EndAt = normalized.EndAt + query.Hours = normalized.Hours + query.Limit = normalized.Limit return query, rangeWasClamped, nil } diff --git a/service/model_performance_test.go b/service/model_performance_test.go index e0574434..ca44a7c3 100644 --- a/service/model_performance_test.go +++ b/service/model_performance_test.go @@ -1,6 +1,7 @@ package service import ( + "context" "errors" "testing" "time" @@ -29,7 +30,7 @@ func TestNormalizeModelPerformanceQueryUsesOneHourDefaultAndClampsWindow(t *test query, clamped, err = normalizeModelPerformanceQuery(ModelPerformanceQuery{Hours: 999}) require.NoError(t, err) require.True(t, clamped) - require.EqualValues(t, maxChannelPerformanceHours*time.Hour/time.Second, query.EndAt-query.StartAt) + require.EqualValues(t, maxPerformanceHours*time.Hour/time.Second, query.EndAt-query.StartAt) } func TestNormalizeModelPerformanceQueryRejectsReversedRange(t *testing.T) { @@ -162,14 +163,14 @@ func TestBuildLegacyModelPerformanceSummaryDistinguishesThroughputStates(t *test func TestQueryModelPerformanceThroughputMarksQueryFailure(t *testing.T) { originalQuery := queryModelPerformanceSummaryRange - queryModelPerformanceSummaryRange = func(int64, int64, []string) (perfmetrics.DetailedSummaryAllResult, error) { + queryModelPerformanceSummaryRange = func(context.Context, int64, int64, []string) (perfmetrics.DetailedSummaryAllResult, error) { return perfmetrics.DetailedSummaryAllResult{}, errors.New("database unavailable") } t.Cleanup(func() { queryModelPerformanceSummaryRange = originalQuery }) - snapshot := queryModelPerformanceThroughput(ModelPerformanceQuery{StartAt: 1, EndAt: 3_601}) + snapshot := queryModelPerformanceThroughput(context.Background(), ModelPerformanceQuery{StartAt: 1, EndAt: 3_601}) require.Equal(t, perfmetrics.CollectionStateQueryFailed, snapshot.State) require.Empty(t, snapshot.ByModel) diff --git a/service/performance_query.go b/service/performance_query.go new file mode 100644 index 00000000..caeee4b5 --- /dev/null +++ b/service/performance_query.go @@ -0,0 +1,67 @@ +package service + +import ( + "errors" + "fmt" + "time" +) + +const ( + defaultPerformanceHours = 1 + maxPerformanceHours = 168 + defaultPerformanceLimit = 100 + maxPerformanceLimit = 200 +) + +type normalizedPerformanceQuery struct { + StartAt int64 + EndAt int64 + Hours int + Limit int +} + +func normalizePerformanceQuery( + startAt int64, + endAt int64, + hours int, + limit int, + invalidQueryError error, +) (normalizedPerformanceQuery, bool, error) { + if invalidQueryError == nil { + invalidQueryError = errors.New("invalid performance query") + } + if endAt <= 0 { + endAt = time.Now().Unix() + } + rangeWasClamped := false + if startAt <= 0 { + if hours <= 0 { + hours = defaultPerformanceHours + } + if hours > maxPerformanceHours { + hours = maxPerformanceHours + rangeWasClamped = true + } + startAt = endAt - int64(hours)*int64(time.Hour/time.Second) + } + if endAt <= startAt { + return normalizedPerformanceQuery{}, false, fmt.Errorf("%w: end must be greater than start", invalidQueryError) + } + if limit <= 0 { + limit = defaultPerformanceLimit + } + if limit > maxPerformanceLimit { + limit = maxPerformanceLimit + } + maxWindow := int64(maxPerformanceHours) * int64(time.Hour/time.Second) + if endAt-startAt > maxWindow { + startAt = endAt - maxWindow + rangeWasClamped = true + } + return normalizedPerformanceQuery{ + StartAt: startAt, + EndAt: endAt, + Hours: hours, + Limit: limit, + }, rangeWasClamped, nil +} diff --git a/web/default/src/features/performance-metrics/types.ts b/web/default/src/features/performance-metrics/types.ts index 9714e7fe..5a58f62f 100644 --- a/web/default/src/features/performance-metrics/types.ts +++ b/web/default/src/features/performance-metrics/types.ts @@ -54,7 +54,7 @@ export type PerformanceCoverage = { bucket_start_at: number bucket_end_at: number bucket_seconds: number - granularity_state?: 'known' | 'unknown' | 'mixed' + granularity_state: 'known' | 'unknown' | 'mixed' approximate: boolean } diff --git a/web/default/src/features/pricing/components/model-details-chart-utils.ts b/web/default/src/features/pricing/components/model-details-chart-utils.ts index 01db9307..02efca43 100644 --- a/web/default/src/features/pricing/components/model-details-chart-utils.ts +++ b/web/default/src/features/pricing/components/model-details-chart-utils.ts @@ -18,7 +18,27 @@ For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API */ import type { LatencyTimePoint, UptimeDayPoint } from '../lib/mock-stats' -export function buildLatencyChartData(series: LatencyTimePoint[]) { +export type LatencyChartDatum = { + time: string + group: string + ttft: number +} + +export type UptimeChartDatum = { + date: string + uptime: number + incidents: number + outage: number +} + +export type UptimeAxisDomain = { + min: number + max: number +} + +export function buildLatencyChartData( + series: LatencyTimePoint[] +): LatencyChartDatum[] { return series.map((point) => ({ time: point.timestamp, group: point.group, @@ -26,7 +46,9 @@ export function buildLatencyChartData(series: LatencyTimePoint[]) { })) } -export function buildUptimeChartData(series: UptimeDayPoint[]) { +export function buildUptimeChartData( + series: UptimeDayPoint[] +): UptimeChartDatum[] { return series.map((point) => ({ date: point.date, uptime: point.uptime_pct, @@ -57,19 +79,19 @@ export function downsampleUptimeSeries( }) } -export function formatChartAxisTime(iso: string): string { +export function formatChartAxisTime(iso: string, locale?: string): string { const date = new Date(iso) if (Number.isNaN(date.getTime())) return iso - return date.toLocaleTimeString(undefined, { + return date.toLocaleTimeString(locale, { hour: '2-digit', minute: '2-digit', }) } -export function formatChartTooltipTime(iso: string): string { +export function formatChartTooltipTime(iso: string, locale?: string): string { const date = new Date(iso) if (Number.isNaN(date.getTime())) return iso - return date.toLocaleString(undefined, { + return date.toLocaleString(locale, { month: 'short', day: 'numeric', hour: '2-digit', @@ -77,7 +99,9 @@ export function formatChartTooltipTime(iso: string): string { }) } -export function getUptimeAxisDomain(series: UptimeDayPoint[]) { +export function getUptimeAxisDomain( + series: UptimeDayPoint[] +): UptimeAxisDomain { const values = series .map((point) => point.uptime_pct) .filter((value) => Number.isFinite(value)) diff --git a/web/default/src/features/pricing/components/model-details-charts.tsx b/web/default/src/features/pricing/components/model-details-charts.tsx index 90884a35..52780d0d 100644 --- a/web/default/src/features/pricing/components/model-details-charts.tsx +++ b/web/default/src/features/pricing/components/model-details-charts.tsx @@ -54,7 +54,7 @@ export function LatencyTrendChart(props: { series: LatencyTimePoint[] className?: string }) { - const { t } = useTranslation() + const { t, i18n } = useTranslation() const { resolvedTheme, themeReady } = useChartTheme() const { textColor, gridColor } = getChartThemeTokens(resolvedTheme) @@ -80,7 +80,8 @@ export function LatencyTrendChart(props: { tooltip: { mark: { title: { - value: (d: { time: string }) => formatChartTooltipTime(d.time), + value: (d: { time: string }) => + formatChartTooltipTime(d.time, i18n.language), }, content: [ { @@ -95,7 +96,7 @@ export function LatencyTrendChart(props: { orient: 'bottom', label: { formatMethod: (val: number | string) => - formatChartAxisTime(String(val)), + formatChartAxisTime(String(val), i18n.language), style: { fill: textColor, fontSize: 10 }, }, tick: { visible: false }, @@ -113,7 +114,7 @@ export function LatencyTrendChart(props: { }, ], } - }, [gridColor, props.series, t, textColor]) + }, [gridColor, i18n.language, props.series, t, textColor]) if (props.series.length === 0) { return ( @@ -153,7 +154,7 @@ export function UptimeTrendChart(props: { series: UptimeDayPoint[] className?: string }) { - const { t } = useTranslation() + const { t, i18n } = useTranslation() const { resolvedTheme, themeReady } = useChartTheme() const { textColor, gridColor } = getChartThemeTokens(resolvedTheme) @@ -188,7 +189,8 @@ export function UptimeTrendChart(props: { tooltip: { mark: { title: { - value: (d: { date: string }) => formatChartTooltipTime(d.date), + value: (d: { date: string }) => + formatChartTooltipTime(d.date, i18n.language), }, content: [ { @@ -211,7 +213,7 @@ export function UptimeTrendChart(props: { orient: 'bottom', label: { formatMethod: (val: number | string) => - formatChartAxisTime(String(val)), + formatChartAxisTime(String(val), i18n.language), style: { fill: textColor, fontSize: 10 }, autoLimit: true, }, @@ -232,7 +234,7 @@ export function UptimeTrendChart(props: { }, ], } - }, [gridColor, props.series, t, textColor]) + }, [gridColor, i18n.language, props.series, t, textColor]) if (props.series.length === 0) { return ( diff --git a/web/default/src/features/pricing/components/model-details-performance.tsx b/web/default/src/features/pricing/components/model-details-performance.tsx index 993444ee..8c4bf4eb 100644 --- a/web/default/src/features/pricing/components/model-details-performance.tsx +++ b/web/default/src/features/pricing/components/model-details-performance.tsx @@ -193,15 +193,27 @@ export function ModelPerformanceDetailsContent(props: { ) } + const aggregateAvailable = aggregate != null const avgTps = aggregate?.avg_tps ?? Number.NaN const avgLatency = aggregate?.avg_latency_ms ?? Number.NaN const successRate = aggregate?.success_rate ?? Number.NaN const incidentCount = uptimeSeries.reduce((s, p) => s + p.incidents, 0) - let intent: 'default' | 'warning' | 'success' = 'warning' - if (successRate >= 99.9) { - intent = 'success' - } else if (successRate >= 99) { - intent = 'default' + let intent: 'default' | 'warning' | 'success' = 'default' + if (aggregateAvailable) { + if (successRate >= 99.9) { + intent = 'success' + } else if (successRate < 99) { + intent = 'warning' + } + } + let successRateHint = t('N/A') + if (aggregateAvailable) { + successRateHint = + incidentCount > 0 + ? t('{{count}} incidents in the last 24 hours', { + count: incidentCount, + }) + : t('No incidents in the last 24 hours') } const headerCellClass = @@ -225,13 +237,7 @@ export function ModelPerformanceDetailsContent(props: { icon={HeartPulse} label={t('Success rate')} value={formatUptimePct(successRate)} - hint={ - incidentCount > 0 - ? t('{{count}} incidents in the last 24 hours', { - count: incidentCount, - }) - : t('No incidents in the last 24 hours') - } + hint={successRateHint} intent={intent} />
diff --git a/web/default/src/features/pricing/components/model-details-uptime-sparkline.tsx b/web/default/src/features/pricing/components/model-details-uptime-sparkline.tsx index 366767c3..a0d0d679 100644 --- a/web/default/src/features/pricing/components/model-details-uptime-sparkline.tsx +++ b/web/default/src/features/pricing/components/model-details-uptime-sparkline.tsx @@ -77,6 +77,26 @@ export function UptimeSparkline(props: UptimeSparklineProps) { const size = props.size ?? 'md' const showOverall = props.showOverall ?? true + const calculatedOverall = useMemo( + () => + props.series.length > 0 + ? props.series.reduce((sum, point) => sum + point.uptime_pct, 0) / + props.series.length + : 0, + [props.series] + ) + const overall = useMemo( + () => + typeof props.overallPct === 'number' && Number.isFinite(props.overallPct) + ? props.overallPct + : calculatedOverall, + [calculatedOverall, props.overallPct] + ) + const displaySeries = useMemo( + () => downsampleUptimeSeries(props.series, size === 'sm' ? 32 : 48), + [props.series, size] + ) + if (props.series.length === 0) { return ( @@ -85,16 +105,6 @@ export function UptimeSparkline(props: UptimeSparklineProps) { ) } - const calculatedOverall = - props.series.reduce((s, p) => s + p.uptime_pct, 0) / props.series.length - const overall = Number.isFinite(props.overallPct) - ? (props.overallPct as number) - : calculatedOverall - const displaySeries = downsampleUptimeSeries( - props.series, - size === 'sm' ? 32 : 48 - ) - const containerHeight = size === 'sm' ? 'h-3.5' : 'h-5' const barWidth = size === 'sm' ? 'w-[3px]' : 'w-1' const gap = size === 'sm' ? 'gap-px' : 'gap-[2px]' diff --git a/web/default/src/features/smart-ops/api.ts b/web/default/src/features/smart-ops/api.ts index 435f53bb..237ef24d 100644 --- a/web/default/src/features/smart-ops/api.ts +++ b/web/default/src/features/smart-ops/api.ts @@ -17,6 +17,7 @@ along with this program. If not, see . For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues */ import { api } from '@/lib/api' +import { MAX_PERFORMANCE_LIMIT } from './lib/filters' import type { ChannelPerformanceData, ChannelPerformanceQuery, @@ -36,7 +37,7 @@ export async function getChannelPerformance( { params: { hours: query.hours, - limit: 200, + limit: MAX_PERFORMANCE_LIMIT, channel_id: query.channelId, model: query.model || undefined, group: query.group || undefined, @@ -78,7 +79,7 @@ export async function getModelPerformance( { params: { hours: query.hours, - limit: 200, + limit: MAX_PERFORMANCE_LIMIT, model: query.model || undefined, group: query.group || undefined, }, diff --git a/web/default/src/features/smart-ops/index.test.tsx b/web/default/src/features/smart-ops/index.test.tsx index a95dbcc1..c3066abd 100644 --- a/web/default/src/features/smart-ops/index.test.tsx +++ b/web/default/src/features/smart-ops/index.test.tsx @@ -25,7 +25,6 @@ import { after, before, describe, test } from 'node:test' import { api } from '@/lib/api' import { formatQuota } from '@/lib/format' import { ProductionPerformance } from './index' -import { getChannelDetailObservation } from './lib/channel-detail' import type { ChannelPerformanceData } from './types' const testEnv = createReactTestEnvironment() @@ -85,7 +84,6 @@ const performanceDataWithErrors: ChannelPerformanceData = { latency_sample_count: 7, observed_success_rate: 70, avg_logged_latency_ms: 1000, - avg_tps: null, last_observed_at: 3600, probe_latency_ms: 100, probe_test_time: 3500, @@ -498,19 +496,6 @@ describe('ProductionPerformance manual queries', () => { } }) - test('uses channel-scoped detail evidence instead of the clicked list row', () => { - const observation = getChannelDetailObservation( - performanceDataWithErrors.items[0], - channelDetailData - ) - - assert.equal( - observation.lastObservedAt, - channelDetailData.summary.last_observed_at - ) - assert.deepEqual(observation.qualityFlags, channelDetailData.quality_flags) - }) - test('exposes an accessible sort menu for every performance metric', async () => { const originalGet = api.get api.get = (async () => ({ diff --git a/web/default/src/features/smart-ops/index.tsx b/web/default/src/features/smart-ops/index.tsx index cb5f91b3..751f1fb9 100644 --- a/web/default/src/features/smart-ops/index.tsx +++ b/web/default/src/features/smart-ops/index.tsx @@ -142,6 +142,14 @@ export function ProductionPerformance() { () => new Intl.NumberFormat(i18n.language), [i18n.language] ) + const dateFormatter = useMemo( + () => + new Intl.DateTimeFormat(i18n.language, { + dateStyle: 'medium', + timeStyle: 'medium', + }), + [i18n.language] + ) const performanceQuery = useMutation({ mutationKey: ['smart-ops', 'channel-performance'], @@ -178,10 +186,7 @@ export function ProductionPerformance() { const formatTimestamp = (timestamp: number | null | undefined) => { if (!timestamp) return '—' - return new Intl.DateTimeFormat(i18n.language, { - dateStyle: 'medium', - timeStyle: 'medium', - }).format(new Date(timestamp * 1000)) + return dateFormatter.format(new Date(timestamp * 1000)) } const qualityLabel = (flag: string) => { @@ -206,6 +211,33 @@ export function ProductionPerformance() { return labels[flag] ?? flag } + let summarySuccessRate = '—' + if (data) { + summarySuccessRate = + data.summary.observed_success_rate == null + ? t('N/A') + : formatUptimePct(data.summary.observed_success_rate) + } + const showPerformanceError = + !performanceQuery.isPending && performanceQuery.isError + const showPerformanceIdle = + !performanceQuery.isPending && + !performanceQuery.isError && + performanceQuery.isIdle + const showPerformanceEmpty = + !performanceQuery.isPending && + !performanceQuery.isError && + !performanceQuery.isIdle && + !data?.items.length + const showPerformanceTable = + !performanceQuery.isPending && + !performanceQuery.isError && + !performanceQuery.isIdle && + Boolean(data?.items.length) + const showDetailError = !detailQuery.isPending && detailQuery.isError + const showDetailData = + !detailQuery.isPending && !detailQuery.isError && detailQuery.data != null + return ( @@ -399,13 +431,7 @@ export function ProductionPerformance() { - {performanceQuery.isPending ? ( - - ) : performanceQuery.isError ? ( + {performanceQuery.isPending && } + {showPerformanceError && ( performanceQuery.mutate(queryParams)} /> - ) : performanceQuery.isIdle ? ( + )} + {showPerformanceIdle && ( @@ -442,7 +468,8 @@ export function ProductionPerformance() { - ) : !data?.items.length ? ( + )} + {showPerformanceEmpty && ( @@ -456,7 +483,8 @@ export function ProductionPerformance() { - ) : ( + )} + {showPerformanceTable && data && (
@@ -529,8 +557,6 @@ export function ProductionPerformance() { {sortedItems.map((item) => ( openDetails(item)} >
@@ -586,10 +612,7 @@ export function ProductionPerformance() { variant='ghost' size='xs' aria-label={`${t('View details')}: ${item.channel_name} · ${item.model_name}`} - onClick={(event) => { - event.stopPropagation() - openDetails(item) - }} + onClick={() => openDetails(item)} > {t('Details')} @@ -685,9 +708,10 @@ export function ProductionPerformance() { )} - {detailQuery.isPending ? ( + {detailQuery.isPending && ( - ) : detailQuery.isError ? ( + )} + {showDetailError && ( detailQuery.mutate(selected.channel_id)} /> - ) : detailQuery.data ? ( + )} + {showDetailData && detailQuery.data && ( - ) : null} + )}
)} @@ -713,6 +738,28 @@ export function ProductionPerformance() { ) } +type SortDirection = 'asc' | 'desc' | null + +function getAriaSort( + direction: SortDirection +): 'ascending' | 'descending' | 'none' { + if (direction === 'asc') return 'ascending' + if (direction === 'desc') return 'descending' + return 'none' +} + +function SortDirectionIcon(props: { + direction: SortDirection +}): React.ReactElement { + if (props.direction === 'desc') { + return
- {performanceQuery.isPending ? ( - - ) : performanceQuery.isError ? ( + {performanceQuery.isPending && } + {showPerformanceError && ( performanceQuery.mutate(queryParams)} /> - ) : performanceQuery.isIdle ? ( + )} + {showPerformanceIdle && ( @@ -499,7 +596,8 @@ export function ModelPerformance() { - ) : !data?.items.length ? ( + )} + {showPerformanceEmpty && ( @@ -513,7 +611,8 @@ export function ModelPerformance() { - ) : ( + )} + {showPerformanceTable && data && (
@@ -593,11 +692,7 @@ export function ModelPerformance() { {sortedItems.map((item) => ( - openDetails(item)} - > + {item.model_name} @@ -637,10 +732,7 @@ export function ModelPerformance() { variant='ghost' size='xs' aria-label={`${t('View details')}: ${item.model_name}`} - onClick={(event) => { - event.stopPropagation() - openDetails(item) - }} + onClick={() => openDetails(item)} > {t('Details')} @@ -759,9 +851,8 @@ export function ModelPerformance() { )} - {detailQuery.isPending ? ( - - ) : detailQuery.isError ? ( + {detailQuery.isPending && } + {showDetailError && ( detailQuery.mutate(selected.model_name)} /> - ) : detailQuery.data ? ( + )} + {showDetailData && detailQuery.data && ( <> - {detailQuery.data.collection_state === - 'collection_disabled' && ( - - - )} - {detailQuery.data.collection_state === 'no_samples' && ( - - - )} - {detailQuery.data.collection_state === 'partial' && ( - - - )} + {detailQuery.data.coverage?.approximate && (
)} @@ -848,6 +896,28 @@ export function ModelPerformance() { ) } +type SortDirection = 'asc' | 'desc' | null + +function getAriaSort( + direction: SortDirection +): 'ascending' | 'descending' | 'none' { + if (direction === 'asc') return 'ascending' + if (direction === 'desc') return 'descending' + return 'none' +} + +function SortDirectionIcon(props: { + direction: SortDirection +}): React.ReactElement { + if (props.direction === 'desc') { + return
{/* Admin Actions */} - {isAdmin && record.status === 'pending' && ( + {isAdmin && isCompletableTopupStatus(record.status) && (
+
+ + {loading ? ( +
+ {Array.from({ length: 3 }).map((_, index) => ( + + ))} +
+ ) : alertsQuery.isError ? ( + void alertsQuery.refetch()} + className='min-h-[240px]' + /> + ) : alerts.length === 0 ? ( + + + + + {t('No active alerts.')} + + {t( + 'This process is not currently reporting a sustained CPU, memory, or disk alert.' + )} + + + + ) : ( +
+ +
+ )} + + + + ) +} diff --git a/web/default/src/features/smart-ops/api.ts b/web/default/src/features/smart-ops/api.ts index 237ef24d..2da83b80 100644 --- a/web/default/src/features/smart-ops/api.ts +++ b/web/default/src/features/smart-ops/api.ts @@ -27,8 +27,16 @@ import type { ModelPerformanceDetailResponse, ModelPerformanceQuery, ModelPerformanceResponse, + SmartOpsAlertsResponse, } from './types' +export async function getSmartOpsAlerts(): Promise { + const response = await api.get( + '/api/smart-ops/alerts' + ) + return response.data +} + export async function getChannelPerformance( query: ChannelPerformanceQuery ): Promise { diff --git a/web/default/src/features/smart-ops/types.ts b/web/default/src/features/smart-ops/types.ts index 6f62c44b..10c2333d 100644 --- a/web/default/src/features/smart-ops/types.ts +++ b/web/default/src/features/smart-ops/types.ts @@ -23,6 +23,24 @@ import type { PerformanceGroup, } from '@/features/performance-metrics/types' +export type SmartOpsAlert = { + key: string + status: 'firing' + severity: 'warning' + component: string + node?: string + current_value: number + threshold: number + observed_at: string + message: string +} + +export type SmartOpsAlertsResponse = { + success: boolean + message?: string + data: SmartOpsAlert[] +} + export type ChannelPerformanceQuery = { hours: number channelId?: number diff --git a/web/default/src/hooks/use-sidebar-config.ts b/web/default/src/hooks/use-sidebar-config.ts index 68207264..e42209a9 100644 --- a/web/default/src/hooks/use-sidebar-config.ts +++ b/web/default/src/hooks/use-sidebar-config.ts @@ -115,6 +115,7 @@ const URL_TO_CONFIG_MAP: Record = { '/redemption-codes': { section: 'admin', module: 'redemption' }, '/subscriptions': { section: 'admin', module: 'subscription' }, '/smart-ops': { section: 'admin', module: 'smart_ops' }, + '/smart-ops/alerts': { section: 'admin', module: 'smart_ops' }, '/smart-ops/production-performance': { section: 'admin', module: 'smart_ops', diff --git a/web/default/src/hooks/use-sidebar-data.ts b/web/default/src/hooks/use-sidebar-data.ts index 36a6a525..9eb21aca 100644 --- a/web/default/src/hooks/use-sidebar-data.ts +++ b/web/default/src/hooks/use-sidebar-data.ts @@ -144,7 +144,7 @@ export function useSidebarData(): SidebarData { }, { title: t('Smart Operations'), - url: '/smart-ops/channel-performance', + url: '/smart-ops/alerts', activeUrls: ['/smart-ops'], icon: HeartPulse, }, diff --git a/web/default/src/i18n/locales/en.json b/web/default/src/i18n/locales/en.json index 0bcf6c90..08d8bb50 100644 --- a/web/default/src/i18n/locales/en.json +++ b/web/default/src/i18n/locales/en.json @@ -5446,6 +5446,19 @@ "No eligible production logs were recorded for this channel in the last 24 hours.": "No eligible production logs were recorded for this channel in the last 24 hours.", "The detail table was truncated to {{count}} rows. The summary still covers all matching eligible logs.": "The detail table was truncated to {{count}} rows. The summary still covers all matching eligible logs.", "Throughput data is partial": "Throughput data is partial", - "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete.": "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete." + "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete.": "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete.", + "Active Alerts": "Active Alerts", + "Alerts requiring administrator attention": "Alerts requiring administrator attention", + "SmartOps reports sustained host pressure here for manual investigation and repair.": "SmartOps reports sustained host pressure here for manual investigation and repair.", + "Resource": "Resource", + "Threshold": "Threshold", + "Observed": "Observed", + "CPU usage": "CPU usage", + "Memory usage": "Memory usage", + "Disk usage": "Disk usage", + "Firing": "Firing", + "We could not load active alerts.": "We could not load active alerts.", + "No active alerts.": "No active alerts.", + "This process is not currently reporting a sustained CPU, memory, or disk alert.": "This process is not currently reporting a sustained CPU, memory, or disk alert." } } diff --git a/web/default/src/i18n/locales/fr.json b/web/default/src/i18n/locales/fr.json index eef71568..22c7e98d 100644 --- a/web/default/src/i18n/locales/fr.json +++ b/web/default/src/i18n/locales/fr.json @@ -5446,6 +5446,19 @@ "No eligible production logs were recorded for this channel in the last 24 hours.": "Aucun journal de production éligible n'a été enregistré pour ce canal au cours des dernières 24 heures.", "The detail table was truncated to {{count}} rows. The summary still covers all matching eligible logs.": "Le tableau détaillé a été limité à {{count}} lignes. Le résumé couvre toujours tous les journaux éligibles correspondants.", "Throughput data is partial": "Données de débit partielles", - "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete.": "Les échantillons stockés et locaux sont affichés, mais les compartiments actifs partagés sont illisibles. Le débit multi-nœuds récent peut être incomplet." + "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete.": "Les échantillons stockés et locaux sont affichés, mais les compartiments actifs partagés sont illisibles. Le débit multi-nœuds récent peut être incomplet.", + "Active Alerts": "Alertes actives", + "Alerts requiring administrator attention": "Alertes nécessitant l’attention d’un administrateur", + "SmartOps reports sustained host pressure here for manual investigation and repair.": "SmartOps signale ici les pressions persistantes sur l’hôte afin qu’elles soient examinées et corrigées manuellement.", + "Resource": "Ressource", + "Threshold": "Seuil", + "Observed": "Observé", + "CPU usage": "Utilisation du processeur", + "Memory usage": "Utilisation de la mémoire", + "Disk usage": "Utilisation du disque", + "Firing": "En cours", + "We could not load active alerts.": "Impossible de charger les alertes actives.", + "No active alerts.": "Aucune alerte active.", + "This process is not currently reporting a sustained CPU, memory, or disk alert.": "Ce processus ne signale actuellement aucune alerte persistante concernant le processeur, la mémoire ou le disque." } } diff --git a/web/default/src/i18n/locales/ja.json b/web/default/src/i18n/locales/ja.json index dc864b5e..7fc1337e 100644 --- a/web/default/src/i18n/locales/ja.json +++ b/web/default/src/i18n/locales/ja.json @@ -5446,6 +5446,19 @@ "No eligible production logs were recorded for this channel in the last 24 hours.": "過去24時間にこのチャネルの対象本番ログは記録されていません。", "The detail table was truncated to {{count}} rows. The summary still covers all matching eligible logs.": "詳細テーブルは{{count}}行に制限されています。概要は一致する対象ログをすべてカバーします。", "Throughput data is partial": "スループットデータが不完全です", - "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete.": "保存済みおよびローカルの性能サンプルを表示していますが、共有アクティブバケットを読み取れませんでした。最近のマルチノードスループットは不完全な可能性があります。" + "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete.": "保存済みおよびローカルの性能サンプルを表示していますが、共有アクティブバケットを読み取れませんでした。最近のマルチノードスループットは不完全な可能性があります。", + "Active Alerts": "アクティブなアラート", + "Alerts requiring administrator attention": "管理者の対応が必要なアラート", + "SmartOps reports sustained host pressure here for manual investigation and repair.": "SmartOps は、手動での調査と復旧のために、継続的なホスト負荷をここに報告します。", + "Resource": "リソース", + "Threshold": "しきい値", + "Observed": "観測時刻", + "CPU usage": "CPU 使用率", + "Memory usage": "メモリ使用率", + "Disk usage": "ディスク使用率", + "Firing": "発報中", + "We could not load active alerts.": "アクティブなアラートを読み込めませんでした。", + "No active alerts.": "アクティブなアラートはありません。", + "This process is not currently reporting a sustained CPU, memory, or disk alert.": "このプロセスは現在、継続的な CPU、メモリ、ディスクのアラートを報告していません。" } } diff --git a/web/default/src/i18n/locales/ru.json b/web/default/src/i18n/locales/ru.json index 3c2520e8..a7780eb7 100644 --- a/web/default/src/i18n/locales/ru.json +++ b/web/default/src/i18n/locales/ru.json @@ -5446,6 +5446,19 @@ "No eligible production logs were recorded for this channel in the last 24 hours.": "За последние 24 часа для этого канала не записано подходящих производственных журналов.", "The detail table was truncated to {{count}} rows. The summary still covers all matching eligible logs.": "Таблица подробностей ограничена {{count}} строками. Сводка по-прежнему охватывает все подходящие журналы.", "Throughput data is partial": "Данные о пропускной способности неполны", - "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete.": "Показаны сохранённые и локальные образцы производительности, но общие активные сегменты не удалось прочитать. Последняя многонодовая пропускная способность может быть неполной." + "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete.": "Показаны сохранённые и локальные образцы производительности, но общие активные сегменты не удалось прочитать. Последняя многонодовая пропускная способность может быть неполной.", + "Active Alerts": "Активные оповещения", + "Alerts requiring administrator attention": "Оповещения, требующие внимания администратора", + "SmartOps reports sustained host pressure here for manual investigation and repair.": "SmartOps сообщает здесь о длительной нагрузке на узел для ручной диагностики и устранения.", + "Resource": "Ресурс", + "Threshold": "Порог", + "Observed": "Время наблюдения", + "CPU usage": "Использование CPU", + "Memory usage": "Использование памяти", + "Disk usage": "Использование диска", + "Firing": "Активно", + "We could not load active alerts.": "Не удалось загрузить активные оповещения.", + "No active alerts.": "Активных оповещений нет.", + "This process is not currently reporting a sustained CPU, memory, or disk alert.": "Этот процесс сейчас не сообщает о длительном оповещении по CPU, памяти или диску." } } diff --git a/web/default/src/i18n/locales/vi.json b/web/default/src/i18n/locales/vi.json index 38fe568f..10f152b5 100644 --- a/web/default/src/i18n/locales/vi.json +++ b/web/default/src/i18n/locales/vi.json @@ -5446,6 +5446,19 @@ "No eligible production logs were recorded for this channel in the last 24 hours.": "Không có nhật ký sản xuất hợp lệ nào được ghi cho kênh này trong 24 giờ qua.", "The detail table was truncated to {{count}} rows. The summary still covers all matching eligible logs.": "Bảng chi tiết bị giới hạn ở {{count}} dòng. Phần tóm tắt vẫn bao phủ tất cả nhật ký hợp lệ phù hợp.", "Throughput data is partial": "Dữ liệu thông lượng không đầy đủ", - "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete.": "Các mẫu hiệu năng đã lưu và cục bộ được hiển thị, nhưng không thể đọc các bucket hoạt động dùng chung. Thông lượng đa nút gần đây có thể không đầy đủ." + "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete.": "Các mẫu hiệu năng đã lưu và cục bộ được hiển thị, nhưng không thể đọc các bucket hoạt động dùng chung. Thông lượng đa nút gần đây có thể không đầy đủ.", + "Active Alerts": "Cảnh báo đang hoạt động", + "Alerts requiring administrator attention": "Cảnh báo cần quản trị viên chú ý", + "SmartOps reports sustained host pressure here for manual investigation and repair.": "SmartOps báo cáo tại đây tình trạng áp lực máy chủ kéo dài để quản trị viên kiểm tra và khắc phục thủ công.", + "Resource": "Tài nguyên", + "Threshold": "Ngưỡng", + "Observed": "Thời điểm quan sát", + "CPU usage": "Mức sử dụng CPU", + "Memory usage": "Mức sử dụng bộ nhớ", + "Disk usage": "Mức sử dụng đĩa", + "Firing": "Đang cảnh báo", + "We could not load active alerts.": "Không thể tải cảnh báo đang hoạt động.", + "No active alerts.": "Không có cảnh báo đang hoạt động.", + "This process is not currently reporting a sustained CPU, memory, or disk alert.": "Tiến trình này hiện không báo cáo cảnh báo CPU, bộ nhớ hoặc đĩa kéo dài." } } diff --git a/web/default/src/i18n/locales/zh.json b/web/default/src/i18n/locales/zh.json index 7c205b75..96fca30a 100644 --- a/web/default/src/i18n/locales/zh.json +++ b/web/default/src/i18n/locales/zh.json @@ -5446,6 +5446,19 @@ "No eligible production logs were recorded for this channel in the last 24 hours.": "最近 24 小时未记录该渠道的符合条件生产日志。", "The detail table was truncated to {{count}} rows. The summary still covers all matching eligible logs.": "详情表已截断为 {{count}} 行，但摘要仍覆盖所有符合条件的匹配日志。", "Throughput data is partial": "吞吐量数据不完整", - "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete.": "当前显示已存储和本地性能样本，但无法读取共享活动桶；最近的多节点吞吐量可能不完整。" + "Stored and local performance samples are shown, but shared active buckets could not be read. Recent multi-node throughput may be incomplete.": "当前显示已存储和本地性能样本，但无法读取共享活动桶；最近的多节点吞吐量可能不完整。", + "Active Alerts": "活动告警", + "Alerts requiring administrator attention": "需要管理员关注的告警", + "SmartOps reports sustained host pressure here for manual investigation and repair.": "SmartOps 会在此报告持续的主机资源压力，供管理员人工排查和修复。", + "Resource": "资源", + "Threshold": "阈值", + "Observed": "观测时间", + "CPU usage": "CPU 使用率", + "Memory usage": "内存使用率", + "Disk usage": "磁盘使用率", + "Firing": "告警中", + "We could not load active alerts.": "无法加载活动告警。", + "No active alerts.": "当前没有活动告警。", + "This process is not currently reporting a sustained CPU, memory, or disk alert.": "当前进程暂未报告持续的 CPU、内存或磁盘告警。" } } diff --git a/web/default/src/routeTree.gen.ts b/web/default/src/routeTree.gen.ts index 7b8bfa48..5092b979 100644 --- a/web/default/src/routeTree.gen.ts +++ b/web/default/src/routeTree.gen.ts @@ -54,6 +54,7 @@ import { Route as AuthenticatedSmartOpsSystemInfoRouteImport } from './routes/_a import { Route as AuthenticatedSmartOpsProductionPerformanceRouteImport } from './routes/_authenticated/smart-ops/production-performance' import { Route as AuthenticatedSmartOpsModelPerformanceRouteImport } from './routes/_authenticated/smart-ops/model-performance' import { Route as AuthenticatedSmartOpsChannelPerformanceRouteImport } from './routes/_authenticated/smart-ops/channel-performance' +import { Route as AuthenticatedSmartOpsAlertsRouteImport } from './routes/_authenticated/smart-ops/alerts' import { Route as AuthenticatedModelsSectionRouteImport } from './routes/_authenticated/models/$section' import { Route as AuthenticatedErrorsErrorRouteImport } from './routes/_authenticated/errors/$error' import { Route as AuthenticatedDashboardSectionRouteImport } from './routes/_authenticated/dashboard/$section' @@ -314,6 +315,12 @@ const AuthenticatedSmartOpsChannelPerformanceRoute = path: '/smart-ops/channel-performance', getParentRoute: () => AuthenticatedRouteRoute, } as any) +const AuthenticatedSmartOpsAlertsRoute = + AuthenticatedSmartOpsAlertsRouteImport.update({ + id: '/smart-ops/alerts', + path: '/smart-ops/alerts', + getParentRoute: () => AuthenticatedRouteRoute, + } as any) const AuthenticatedModelsSectionRoute = AuthenticatedModelsSectionRouteImport.update({ id: '/models/$section', @@ -457,6 +464,7 @@ export interface FileRoutesByFullPath { '/dashboard/$section': typeof AuthenticatedDashboardSectionRoute '/errors/$error': typeof AuthenticatedErrorsErrorRoute '/models/$section': typeof AuthenticatedModelsSectionRoute + '/smart-ops/alerts': typeof AuthenticatedSmartOpsAlertsRoute '/smart-ops/channel-performance': typeof AuthenticatedSmartOpsChannelPerformanceRoute '/smart-ops/model-performance': typeof AuthenticatedSmartOpsModelPerformanceRoute '/smart-ops/production-performance': typeof AuthenticatedSmartOpsProductionPerformanceRoute @@ -520,6 +528,7 @@ export interface FileRoutesByTo { '/dashboard/$section': typeof AuthenticatedDashboardSectionRoute '/errors/$error': typeof AuthenticatedErrorsErrorRoute '/models/$section': typeof AuthenticatedModelsSectionRoute + '/smart-ops/alerts': typeof AuthenticatedSmartOpsAlertsRoute '/smart-ops/channel-performance': typeof AuthenticatedSmartOpsChannelPerformanceRoute '/smart-ops/model-performance': typeof AuthenticatedSmartOpsModelPerformanceRoute '/smart-ops/production-performance': typeof AuthenticatedSmartOpsProductionPerformanceRoute @@ -587,6 +596,7 @@ export interface FileRoutesById { '/_authenticated/dashboard/$section': typeof AuthenticatedDashboardSectionRoute '/_authenticated/errors/$error': typeof AuthenticatedErrorsErrorRoute '/_authenticated/models/$section': typeof AuthenticatedModelsSectionRoute + '/_authenticated/smart-ops/alerts': typeof AuthenticatedSmartOpsAlertsRoute '/_authenticated/smart-ops/channel-performance': typeof AuthenticatedSmartOpsChannelPerformanceRoute '/_authenticated/smart-ops/model-performance': typeof AuthenticatedSmartOpsModelPerformanceRoute '/_authenticated/smart-ops/production-performance': typeof AuthenticatedSmartOpsProductionPerformanceRoute @@ -653,6 +663,7 @@ export interface FileRouteTypes { | '/dashboard/$section' | '/errors/$error' | '/models/$section' + | '/smart-ops/alerts' | '/smart-ops/channel-performance' | '/smart-ops/model-performance' | '/smart-ops/production-performance' @@ -716,6 +727,7 @@ export interface FileRouteTypes { | '/dashboard/$section' | '/errors/$error' | '/models/$section' + | '/smart-ops/alerts' | '/smart-ops/channel-performance' | '/smart-ops/model-performance' | '/smart-ops/production-performance' @@ -782,6 +794,7 @@ export interface FileRouteTypes { | '/_authenticated/dashboard/$section' | '/_authenticated/errors/$error' | '/_authenticated/models/$section' + | '/_authenticated/smart-ops/alerts' | '/_authenticated/smart-ops/channel-performance' | '/_authenticated/smart-ops/model-performance' | '/_authenticated/smart-ops/production-performance' @@ -1155,6 +1168,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof AuthenticatedSmartOpsChannelPerformanceRouteImport parentRoute: typeof AuthenticatedRouteRoute } + '/_authenticated/smart-ops/alerts': { + id: '/_authenticated/smart-ops/alerts' + path: '/smart-ops/alerts' + fullPath: '/smart-ops/alerts' + preLoaderRoute: typeof AuthenticatedSmartOpsAlertsRouteImport + parentRoute: typeof AuthenticatedRouteRoute + } '/_authenticated/models/$section': { id: '/_authenticated/models/$section' path: '/models/$section' @@ -1381,6 +1401,7 @@ interface AuthenticatedRouteRouteChildren { AuthenticatedDashboardSectionRoute: typeof AuthenticatedDashboardSectionRoute AuthenticatedErrorsErrorRoute: typeof AuthenticatedErrorsErrorRoute AuthenticatedModelsSectionRoute: typeof AuthenticatedModelsSectionRoute + AuthenticatedSmartOpsAlertsRoute: typeof AuthenticatedSmartOpsAlertsRoute AuthenticatedSmartOpsChannelPerformanceRoute: typeof AuthenticatedSmartOpsChannelPerformanceRoute AuthenticatedSmartOpsModelPerformanceRoute: typeof AuthenticatedSmartOpsModelPerformanceRoute AuthenticatedSmartOpsProductionPerformanceRoute: typeof AuthenticatedSmartOpsProductionPerformanceRoute @@ -1408,6 +1429,7 @@ const AuthenticatedRouteRouteChildren: AuthenticatedRouteRouteChildren = { AuthenticatedDashboardSectionRoute: AuthenticatedDashboardSectionRoute, AuthenticatedErrorsErrorRoute: AuthenticatedErrorsErrorRoute, AuthenticatedModelsSectionRoute: AuthenticatedModelsSectionRoute, + AuthenticatedSmartOpsAlertsRoute: AuthenticatedSmartOpsAlertsRoute, AuthenticatedSmartOpsChannelPerformanceRoute: AuthenticatedSmartOpsChannelPerformanceRoute, AuthenticatedSmartOpsModelPerformanceRoute: diff --git a/web/default/src/routes/_authenticated/smart-ops/alerts.tsx b/web/default/src/routes/_authenticated/smart-ops/alerts.tsx new file mode 100644 index 00000000..4e6481f3 --- /dev/null +++ b/web/default/src/routes/_authenticated/smart-ops/alerts.tsx @@ -0,0 +1,32 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import { createFileRoute, redirect } from '@tanstack/react-router' +import { useAuthStore } from '@/stores/auth-store' +import { ROLE } from '@/lib/roles' +import { ActiveAlerts } from '@/features/smart-ops/alerts' + +export const Route = createFileRoute('/_authenticated/smart-ops/alerts')({ + beforeLoad: () => { + const { auth } = useAuthStore.getState() + if (!auth.user || auth.user.role < ROLE.ADMIN) { + throw redirect({ to: '/403' }) + } + }, + component: ActiveAlerts, +}) From 1fe224a20b9fd2b83904da314b14c2ed27f579af Mon Sep 17 00:00:00 2001 From: CSCITech Date: Sat, 22 Aug 2026 23:47:56 +0800 Subject: [PATCH 26/28] add features --- service/smart_ops_alert.go | 90 +++++++++++++--- service/smart_ops_alert_test.go | 57 ++++++++++ .../src/features/smart-ops/alerts.test.tsx | 100 ++++++++++++++++-- web/default/src/features/smart-ops/alerts.tsx | 33 +++--- web/default/src/i18n/locales/fr.json | 2 +- 5 files changed, 249 insertions(+), 33 deletions(-) diff --git a/service/smart_ops_alert.go b/service/smart_ops_alert.go index b7588f74..3f61dc80 100644 --- a/service/smart_ops_alert.go +++ b/service/smart_ops_alert.go @@ -3,6 +3,7 @@ package service import ( "context" "fmt" + "hash/fnv" "sort" "sync" "time" @@ -19,6 +20,7 @@ const ( smartOpsAlertSampleInterval = 5 * time.Second smartOpsAlertRequiredSamples = 2 smartOpsAlertQueueSize = 24 + smartOpsAlertWorkerCount = 3 ) var smartOpsAlertRetryDelays = []time.Duration{ @@ -315,9 +317,78 @@ type smartOpsAlertRecipient struct { Setting dto.UserSetting } +type smartOpsAlertNotificationPool struct { + queues []chan SmartOpsAlert + deliver func(SmartOpsAlert) error + closeOnce sync.Once + waitGroup sync.WaitGroup +} + +func newSmartOpsAlertNotificationPool(workerCount, queueSize int, deliver func(SmartOpsAlert) error) *smartOpsAlertNotificationPool { + if workerCount < 1 { + workerCount = 1 + } + if queueSize < workerCount { + queueSize = workerCount + } + + pool := &smartOpsAlertNotificationPool{ + queues: make([]chan SmartOpsAlert, workerCount), + deliver: deliver, + } + baseCapacity := queueSize / workerCount + extraCapacity := queueSize % workerCount + for worker := range workerCount { + capacity := baseCapacity + if worker < extraCapacity { + capacity++ + } + pool.queues[worker] = make(chan SmartOpsAlert, capacity) + pool.waitGroup.Add(1) + go pool.runWorker(pool.queues[worker]) + } + return pool +} + +func (pool *smartOpsAlertNotificationPool) runWorker(queue <-chan SmartOpsAlert) { + defer pool.waitGroup.Done() + for alert := range queue { + if err := pool.deliver(alert); err != nil { + common.SysLog(fmt.Sprintf("failed to deliver smart ops alert notification: %s", err.Error())) + } + } +} + +func (pool *smartOpsAlertNotificationPool) workerIndex(alert SmartOpsAlert) int { + hasher := fnv.New32a() + _, _ = hasher.Write([]byte(alert.Node)) + _, _ = hasher.Write([]byte{0}) + _, _ = hasher.Write([]byte(alert.Key)) + return int(hasher.Sum32() % uint32(len(pool.queues))) +} + +func (pool *smartOpsAlertNotificationPool) enqueue(alert SmartOpsAlert) bool { + queue := pool.queues[pool.workerIndex(alert)] + select { + case queue <- alert: + return true + default: + return false + } +} + +func (pool *smartOpsAlertNotificationPool) close() { + pool.closeOnce.Do(func() { + for _, queue := range pool.queues { + close(queue) + } + pool.waitGroup.Wait() + }) +} + var smartOpsAlertNotificationQueue struct { sync.Once - channel chan SmartOpsAlert + pool *smartOpsAlertNotificationPool } var smartOpsAlertLoadRecipients = loadSmartOpsAlertRecipients @@ -328,19 +399,14 @@ var smartOpsAlertSendToRecipient = func(recipient smartOpsAlertRecipient, data d func enqueueSmartOpsAlertNotification(alert SmartOpsAlert) { smartOpsAlertNotificationQueue.Do(func() { - smartOpsAlertNotificationQueue.channel = make(chan SmartOpsAlert, smartOpsAlertQueueSize) - go func() { - for queuedAlert := range smartOpsAlertNotificationQueue.channel { - if err := deliverSmartOpsAlertNotification(queuedAlert); err != nil { - common.SysLog(fmt.Sprintf("failed to deliver smart ops alert notification: %s", err.Error())) - } - } - }() + smartOpsAlertNotificationQueue.pool = newSmartOpsAlertNotificationPool( + smartOpsAlertWorkerCount, + smartOpsAlertQueueSize, + deliverSmartOpsAlertNotification, + ) }) - select { - case smartOpsAlertNotificationQueue.channel <- alert: - default: + if !smartOpsAlertNotificationQueue.pool.enqueue(alert) { common.SysLog(fmt.Sprintf("smart ops alert notification queue is full, dropping %s for node %s", alert.Key, alert.Node)) } } diff --git a/service/smart_ops_alert_test.go b/service/smart_ops_alert_test.go index 75ef94e4..3e2ba9a5 100644 --- a/service/smart_ops_alert_test.go +++ b/service/smart_ops_alert_test.go @@ -218,3 +218,60 @@ func TestDeliverSmartOpsAlertRetriesRecipientLookup(t *testing.T) { require.Equal(t, 2, loadAttempts) require.Equal(t, 1, sent) } + +func TestSmartOpsAlertNotificationPoolRunsIndependentIncidentsConcurrentlyAndPreservesOrder(t *testing.T) { + firstStarted := make(chan struct{}) + releaseFirst := make(chan struct{}) + independentDelivered := make(chan struct{}, 1) + incidentOrder := make(chan string, 2) + + pool := newSmartOpsAlertNotificationPool(2, 4, func(alert SmartOpsAlert) error { + if alert.Key == "system_cpu" && alert.Node == "node-a" { + if alert.Status == smartOpsAlertStatusFiring { + close(firstStarted) + <-releaseFirst + } + incidentOrder <- alert.Status + return nil + } + independentDelivered <- struct{}{} + return nil + }) + t.Cleanup(pool.close) + + firing := SmartOpsAlert{Key: "system_cpu", Node: "node-a", Status: smartOpsAlertStatusFiring} + resolved := firing + resolved.Status = smartOpsAlertStatusResolved + require.Equal(t, pool.workerIndex(firing), pool.workerIndex(resolved), "one incident must stay on one ordered worker") + + independent := SmartOpsAlert{Key: "system_memory", Node: "node-b", Status: smartOpsAlertStatusFiring} + if pool.workerIndex(independent) == pool.workerIndex(firing) { + independent = SmartOpsAlert{Key: "system_disk", Node: "node-c", Status: smartOpsAlertStatusFiring} + } + require.NotEqual(t, pool.workerIndex(firing), pool.workerIndex(independent), "the regression fixture must use another worker") + + require.True(t, pool.enqueue(firing)) + select { + case <-firstStarted: + case <-time.After(time.Second): + t.Fatal("the first incident did not start delivery") + } + + require.True(t, pool.enqueue(resolved)) + require.True(t, pool.enqueue(independent)) + select { + case <-independentDelivered: + case <-time.After(time.Second): + t.Fatal("an unrelated alert was blocked behind a slow incident") + } + + close(releaseFirst) + for _, expected := range []string{smartOpsAlertStatusFiring, smartOpsAlertStatusResolved} { + select { + case actual := <-incidentOrder: + require.Equal(t, expected, actual) + case <-time.After(time.Second): + t.Fatalf("timed out waiting for %s delivery", expected) + } + } +} diff --git a/web/default/src/features/smart-ops/alerts.test.tsx b/web/default/src/features/smart-ops/alerts.test.tsx index 2ea776ed..d85570b8 100644 --- a/web/default/src/features/smart-ops/alerts.test.tsx +++ b/web/default/src/features/smart-ops/alerts.test.tsx @@ -18,19 +18,29 @@ For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API */ import { QueryClient, QueryClientProvider } from '@tanstack/react-query' import { createReactTestEnvironment } from '@/test/react' -import { waitFor } from '@testing-library/react' +import { waitFor, within } from '@testing-library/react' import assert from 'node:assert/strict' import { after, before, describe, test } from 'node:test' import { api } from '@/lib/api' import { ActiveAlerts } from './alerts' -const testEnv = createReactTestEnvironment() +const LOAD_ERROR_KEY = 'We could not load active alerts.' +const testEnv = createReactTestEnvironment({ + resources: { + en: { translation: { [LOAD_ERROR_KEY]: LOAD_ERROR_KEY } }, + fr: { + translation: { + [LOAD_ERROR_KEY]: 'Impossible de charger les alertes actives.', + }, + }, + }, +}) before(() => testEnv.setup()) after(() => testEnv.teardown()) -function createQueryClient() { +function createQueryClient(): QueryClient { return new QueryClient({ defaultOptions: { queries: { retry: false }, @@ -89,6 +99,47 @@ describe('SmartOps active alerts', () => { } }) + test('formats percentages with the active locale', async () => { + const originalGet = api.get + await testEnv.i18n.changeLanguage('fr') + api.get = (async () => ({ + data: { + success: true, + data: [ + { + key: 'system_memory', + status: 'firing', + severity: 'warning', + component: 'system', + node: 'node-fr', + current_value: 95.25, + threshold: 90, + observed_at: '2026-08-22T08:00:00Z', + message: 'Memory usage exceeded the threshold', + }, + ], + }, + })) as typeof api.get + + const queryClient = createQueryClient() + const view = await testEnv.render( + + + + ) + + try { + await waitFor(() => { + assert.ok((view.container.textContent ?? '').includes('95,3%')) + }) + } finally { + api.get = originalGet + queryClient.clear() + await view.unmount() + await testEnv.i18n.changeLanguage('en') + } + }) + test('shows a healthy empty state when no incident is active', async () => { const originalGet = api.get api.get = (async () => ({ @@ -115,9 +166,39 @@ describe('SmartOps active alerts', () => { } }) + test('shows a localized fallback for an unsuccessful response', async () => { + const originalGet = api.get + await testEnv.i18n.changeLanguage('fr') + api.get = (async () => ({ + data: { success: false, data: [] }, + })) as typeof api.get + + const queryClient = createQueryClient() + const view = await testEnv.render( + + + + ) + + try { + await waitFor(() => { + const text = view.container.textContent ?? '' + assert.ok(text.includes('Impossible de charger les alertes actives.')) + assert.ok(!text.includes(LOAD_ERROR_KEY)) + }) + } finally { + api.get = originalGet + queryClient.clear() + await view.unmount() + await testEnv.i18n.changeLanguage('en') + } + }) + test('shows a retryable error state when the alert endpoint fails', async () => { const originalGet = api.get + let requestCount = 0 api.get = (async () => { + requestCount += 1 throw new Error('temporary alert endpoint failure') }) as typeof api.get @@ -129,16 +210,19 @@ describe('SmartOps active alerts', () => { ) try { + let retryButton: HTMLElement | undefined await waitFor(() => { const text = view.container.textContent ?? '' assert.ok(text.includes('We could not load active alerts.')) assert.ok(text.includes('temporary alert endpoint failure')) - assert.ok( - Array.from(view.container.querySelectorAll('button')).some( - (button) => button.textContent?.trim() === 'Retry' - ) - ) + retryButton = within(view.container).getByRole('button', { + name: 'Retry', + }) }) + assert.ok(retryButton) + const requestsBeforeRetry = requestCount + await view.click(retryButton) + await waitFor(() => assert.ok(requestCount > requestsBeforeRetry)) } finally { api.get = originalGet queryClient.clear() diff --git a/web/default/src/features/smart-ops/alerts.tsx b/web/default/src/features/smart-ops/alerts.tsx index f1ac8a90..9f53e8b0 100644 --- a/web/default/src/features/smart-ops/alerts.tsx +++ b/web/default/src/features/smart-ops/alerts.tsx @@ -16,6 +16,7 @@ along with this program. If not, see . For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues */ +import type { ReactElement } from 'react' import { useQuery } from '@tanstack/react-query' import { RefreshCw, ShieldCheck, TriangleAlert } from 'lucide-react' import { useTranslation } from 'react-i18next' @@ -52,16 +53,23 @@ const RESOURCE_LABEL: Record = { system_disk: 'Disk usage', } -function formatPercent(value: number) { - return `${value.toFixed(1)}%` +function formatPercent(value: number, locale: string): string { + return `${new Intl.NumberFormat(locale, { + minimumFractionDigits: 1, + maximumFractionDigits: 1, + }).format(value)}%` } -function getObservedAtMilliseconds(alert: SmartOpsAlert) { +function getObservedAtMilliseconds(alert: SmartOpsAlert): number | undefined { const timestamp = Date.parse(alert.observed_at) return Number.isNaN(timestamp) ? undefined : timestamp } -function ActiveAlertsTable({ alerts }: { alerts: SmartOpsAlert[] }) { +interface ActiveAlertsTableProps { + alerts: SmartOpsAlert[] +} + +function ActiveAlertsTable(props: ActiveAlertsTableProps): ReactElement { const { t, i18n } = useTranslation() return ( @@ -84,7 +92,7 @@ function ActiveAlertsTable({ alerts }: { alerts: SmartOpsAlert[] }) { - {alerts.map((alert) => { + {props.alerts.map((alert) => { const observedAt = getObservedAtMilliseconds(alert) return ( @@ -105,10 +113,10 @@ function ActiveAlertsTable({ alerts }: { alerts: SmartOpsAlert[] }) { {alert.node || t('Unknown')} - {formatPercent(alert.current_value)} + {formatPercent(alert.current_value, i18n.language)} - {formatPercent(alert.threshold)} + {formatPercent(alert.threshold, i18n.language)} { + queryKey: ['smart-ops', 'active-alerts', loadErrorMessage], + queryFn: async (): Promise => { const response = await getSmartOpsAlerts() if (!response.success || !Array.isArray(response.data)) { - throw new Error(response.message || 'We could not load active alerts.') + throw new Error(response.message || loadErrorMessage) } return response.data }, @@ -207,7 +216,7 @@ export function ActiveAlerts() {
) : alertsQuery.isError ? ( Date: Sun, 23 Aug 2026 00:22:10 +0800 Subject: [PATCH 27/28] add features --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 04ad5da9..5523f932 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/MAX-API-Next/MAX-API -// +heroku goVersion go1.18 +// +heroku goVersion go1.25 go 1.25.1 require ( From 7adaa40057b88bd2af8ea736485f5137aa47b8c2 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Sun, 23 Aug 2026 16:20:23 +0800 Subject: [PATCH 28/28] add features --- common/constants.go | 8 +++--- common/init.go | 17 ++++++----- common/init_rate_limit_test.go | 52 ++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 11 deletions(-) create mode 100644 common/init_rate_limit_test.go diff --git a/common/constants.go b/common/constants.go index d4385279..0943455f 100644 --- a/common/constants.go +++ b/common/constants.go @@ -226,15 +226,15 @@ var ( GlobalWebRateLimitDuration int64 CriticalRateLimitEnable bool - CriticalRateLimitNum = 20 + CriticalRateLimitNum = 200 CriticalRateLimitDuration int64 = 20 * 60 - CriticalRouteRateLimitNum = 20 + CriticalRouteRateLimitNum = 200 CriticalRouteRateLimitDuration int64 = 20 * 60 // Password login is limited by account as well as the shared API bucket. // This prevents a shared NAT address from making unrelated users unable to log in. LoginRateLimitEnable bool - LoginRateLimitNum = 10 + LoginRateLimitNum = 100 LoginRateLimitDuration int64 = 15 * 60 UploadRateLimitNum = 10 @@ -245,7 +245,7 @@ var ( // Per-user search rate limit (applies after authentication, keyed by user ID) SearchRateLimitEnable = true - SearchRateLimitNum = 10 + SearchRateLimitNum = 100 SearchRateLimitDuration int64 = 60 ) diff --git a/common/init.go b/common/init.go index 8720cdc1..240b7e6a 100644 --- a/common/init.go +++ b/common/init.go @@ -127,29 +127,32 @@ func InitEnv() { GeminiSafetySetting = GetEnvOrDefaultString("GEMINI_SAFETY_SETTING", "BLOCK_NONE") CohereSafetySetting = GetEnvOrDefaultString("COHERE_SAFETY_SETTING", "NONE") - // Initialize rate limit variables + initRateLimitEnv() + initConstantEnv() +} + +func initRateLimitEnv() { GlobalApiRateLimitEnable = GetEnvOrDefaultBool("GLOBAL_API_RATE_LIMIT_ENABLE", true) GlobalApiRateLimitNum = GetEnvOrDefault("GLOBAL_API_RATE_LIMIT", 720) GlobalApiRateLimitDuration = int64(GetEnvOrDefault("GLOBAL_API_RATE_LIMIT_DURATION", 180)) GlobalWebRateLimitEnable = GetEnvOrDefaultBool("GLOBAL_WEB_RATE_LIMIT_ENABLE", true) - GlobalWebRateLimitNum = GetEnvOrDefault("GLOBAL_WEB_RATE_LIMIT", 60) + GlobalWebRateLimitNum = GetEnvOrDefault("GLOBAL_WEB_RATE_LIMIT", 600) GlobalWebRateLimitDuration = int64(GetEnvOrDefault("GLOBAL_WEB_RATE_LIMIT_DURATION", 180)) CriticalRateLimitEnable = GetEnvOrDefaultBool("CRITICAL_RATE_LIMIT_ENABLE", true) - CriticalRateLimitNum = GetEnvOrDefault("CRITICAL_RATE_LIMIT", 20) + CriticalRateLimitNum = GetEnvOrDefault("CRITICAL_RATE_LIMIT", 200) CriticalRateLimitDuration = int64(GetEnvOrDefault("CRITICAL_RATE_LIMIT_DURATION", 20*60)) - CriticalRouteRateLimitNum = GetEnvOrDefault("CRITICAL_ROUTE_RATE_LIMIT", 20) + CriticalRouteRateLimitNum = GetEnvOrDefault("CRITICAL_ROUTE_RATE_LIMIT", 200) CriticalRouteRateLimitDuration = int64(GetEnvOrDefault("CRITICAL_ROUTE_RATE_LIMIT_DURATION", 20*60)) LoginRateLimitEnable = GetEnvOrDefaultBool("LOGIN_RATE_LIMIT_ENABLE", true) - LoginRateLimitNum = GetEnvOrDefault("LOGIN_RATE_LIMIT", 10) + LoginRateLimitNum = GetEnvOrDefault("LOGIN_RATE_LIMIT", 100) LoginRateLimitDuration = int64(GetEnvOrDefault("LOGIN_RATE_LIMIT_DURATION", 15*60)) SearchRateLimitEnable = GetEnvOrDefaultBool("SEARCH_RATE_LIMIT_ENABLE", true) - SearchRateLimitNum = GetEnvOrDefault("SEARCH_RATE_LIMIT", 10) + SearchRateLimitNum = GetEnvOrDefault("SEARCH_RATE_LIMIT", 100) SearchRateLimitDuration = int64(GetEnvOrDefault("SEARCH_RATE_LIMIT_DURATION", 60)) - initConstantEnv() } func initConstantEnv() { diff --git a/common/init_rate_limit_test.go b/common/init_rate_limit_test.go new file mode 100644 index 00000000..18965a95 --- /dev/null +++ b/common/init_rate_limit_test.go @@ -0,0 +1,52 @@ +package common + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRateLimitEnvironmentDefaults(t *testing.T) { + envKeys := []string{ + "GLOBAL_API_RATE_LIMIT_ENABLE", + "GLOBAL_API_RATE_LIMIT", + "GLOBAL_API_RATE_LIMIT_DURATION", + "GLOBAL_WEB_RATE_LIMIT_ENABLE", + "GLOBAL_WEB_RATE_LIMIT", + "GLOBAL_WEB_RATE_LIMIT_DURATION", + "CRITICAL_RATE_LIMIT_ENABLE", + "CRITICAL_RATE_LIMIT", + "CRITICAL_RATE_LIMIT_DURATION", + "CRITICAL_ROUTE_RATE_LIMIT", + "CRITICAL_ROUTE_RATE_LIMIT_DURATION", + "LOGIN_RATE_LIMIT_ENABLE", + "LOGIN_RATE_LIMIT", + "LOGIN_RATE_LIMIT_DURATION", + "SEARCH_RATE_LIMIT_ENABLE", + "SEARCH_RATE_LIMIT", + "SEARCH_RATE_LIMIT_DURATION", + } + for _, key := range envKeys { + t.Setenv(key, "") + } + + initRateLimitEnv() + + require.True(t, GlobalApiRateLimitEnable) + require.Equal(t, 720, GlobalApiRateLimitNum) + require.EqualValues(t, 180, GlobalApiRateLimitDuration) + require.True(t, GlobalWebRateLimitEnable) + require.Equal(t, 600, GlobalWebRateLimitNum) + require.EqualValues(t, 180, GlobalWebRateLimitDuration) + require.True(t, CriticalRateLimitEnable) + require.Equal(t, 200, CriticalRateLimitNum) + require.EqualValues(t, 20*60, CriticalRateLimitDuration) + require.Equal(t, 200, CriticalRouteRateLimitNum) + require.EqualValues(t, 20*60, CriticalRouteRateLimitDuration) + require.True(t, LoginRateLimitEnable) + require.Equal(t, 100, LoginRateLimitNum) + require.EqualValues(t, 15*60, LoginRateLimitDuration) + require.True(t, SearchRateLimitEnable) + require.Equal(t, 100, SearchRateLimitNum) + require.EqualValues(t, 60, SearchRateLimitDuration) +}