diff --git a/apisix/plugins/ai-protocols/anthropic-messages.lua b/apisix/plugins/ai-protocols/anthropic-messages.lua index 75971e1727a7..827154cc3291 100644 --- a/apisix/plugins/ai-protocols/anthropic-messages.lua +++ b/apisix/plugins/ai-protocols/anthropic-messages.lua @@ -29,10 +29,51 @@ local sse = require("apisix.plugins.ai-transport.sse") local table = table local type = type local ipairs = ipairs +local pairs = pairs local _M = {} +-- Anthropic reports cache tokens alongside input_tokens rather than inside it, +-- so add them back to get a prompt total with the same meaning as OpenAI's +-- prompt_tokens. Mirrors LiteLLM's AnthropicConfig.calculate_usage. +local function to_usage(raw) + local cache_read = raw.cache_read_input_tokens or 0 + local cache_creation = raw.cache_creation_input_tokens or 0 + local prompt = (raw.input_tokens or 0) + cache_read + cache_creation + local completion = raw.output_tokens or 0 + return { + prompt_tokens = prompt, + completion_tokens = completion, + total_tokens = prompt + completion, + cache_read_input_tokens = cache_read, + cache_creation_input_tokens = cache_creation, + reasoning_tokens = 0, + } +end + + +-- message_delta usage is cumulative but may omit fields reported at +-- message_start (and some providers only report cache tokens there), so +-- merge the raw fields across events before computing usage. A later event +-- overwrites a field rather than adding to it, because the values are +-- cumulative: summing them would double-count. +local function merge_usage_fields(state, raw) + if type(state) ~= "table" then + return raw + end + local acc = state.anthropic_usage + if not acc then + acc = {} + state.anthropic_usage = acc + end + for k, v in pairs(raw) do + acc[k] = v + end + return acc +end + + --- Detect whether the request matches the Anthropic Messages API format. -- Uses URI suffix (/v1/messages) and body (valid JSON table). function _M.matches(body, ctx) @@ -83,12 +124,7 @@ function _M.parse_sse_event(event, ctx, state) if type(data.usage) == "table" then return { type = "usage", - usage = { - prompt_tokens = data.usage.input_tokens or 0, - completion_tokens = data.usage.output_tokens or 0, - total_tokens = (data.usage.input_tokens or 0) - + (data.usage.output_tokens or 0), - }, + usage = to_usage(merge_usage_fields(state, data.usage)), raw_usage = data.usage, } end @@ -106,14 +142,7 @@ function _M.parse_sse_event(event, ctx, state) local usage = data.message.usage return { type = "usage", - usage = { - prompt_tokens = usage.input_tokens or 0, - completion_tokens = usage.output_tokens or 0, - total_tokens = (usage.input_tokens or 0) + (usage.output_tokens or 0), - cache_read_input_tokens = usage.cache_read_input_tokens or 0, - cache_creation_input_tokens = usage.cache_creation_input_tokens or 0, - reasoning_tokens = 0, - }, + usage = to_usage(merge_usage_fields(state, usage)), raw_usage = usage, } end @@ -174,16 +203,7 @@ function _M.extract_usage(res_body) return nil, nil end local raw = res_body.usage - local prompt = raw.input_tokens or 0 - local completion = raw.output_tokens or 0 - return { - prompt_tokens = prompt, - completion_tokens = completion, - total_tokens = prompt + completion, - cache_read_input_tokens = raw.cache_read_input_tokens or 0, - cache_creation_input_tokens = raw.cache_creation_input_tokens or 0, - reasoning_tokens = 0, - }, raw + return to_usage(raw), raw end diff --git a/apisix/plugins/ai-protocols/converters/anthropic-messages-to-openai-chat.lua b/apisix/plugins/ai-protocols/converters/anthropic-messages-to-openai-chat.lua index 6657db881e36..0ba0779e6ca7 100644 --- a/apisix/plugins/ai-protocols/converters/anthropic-messages-to-openai-chat.lua +++ b/apisix/plugins/ai-protocols/converters/anthropic-messages-to-openai-chat.lua @@ -25,6 +25,7 @@ -- fields never reach the upstream provider. local core = require("apisix.core") +local openai_chat = require("apisix.plugins.ai-protocols.openai-chat") local resty_sha256 = require("resty.sha256") local to_hex = require("resty.string").to_hex local table = table @@ -787,15 +788,16 @@ function _M.convert_response(res_body, ctx) usage.output_tokens = completion_tokens - if type(details) == "table" then - local cached = details.cached_tokens or 0 - usage.input_tokens = math_max(0, prompt_tokens - cached) + -- Anthropic input_tokens excludes both cache reads and cache writes, + -- while OpenAI prompt_tokens includes them (same as LiteLLM). + local cached = openai_chat.cache_read_tokens(res_body.usage) + local creation = openai_chat.cache_creation_tokens(res_body.usage) + usage.input_tokens = math_max(0, prompt_tokens - cached - creation) + if type(details) == "table" or cached > 0 then usage.cache_read_input_tokens = cached - if details.cache_creation_input_tokens then - usage.cache_creation_input_tokens = details.cache_creation_input_tokens - end - else - usage.input_tokens = prompt_tokens + end + if creation > 0 then + usage.cache_creation_input_tokens = creation end end @@ -830,22 +832,18 @@ local function openai_to_anthropic_sse(openai_chunk, state, tool_name_map) if state.pending_stop then local message_delta = state.pending_message_delta if type(openai_chunk.usage) == "table" and not message_delta.usage then - local details = openai_chunk.usage.prompt_tokens_details local prompt_tokens = openai_chunk.usage.prompt_tokens or 0 - local cached = 0 - if type(details) == "table" then - cached = details.cached_tokens or 0 - end + local cached = openai_chat.cache_read_tokens(openai_chunk.usage) + local creation = openai_chat.cache_creation_tokens(openai_chunk.usage) message_delta.usage = { - input_tokens = math_max(0, prompt_tokens - cached), + input_tokens = math_max(0, prompt_tokens - cached - creation), output_tokens = openai_chunk.usage.completion_tokens or 0, } if cached > 0 then message_delta.usage.cache_read_input_tokens = cached end - if type(details) == "table" and details.cache_creation_input_tokens then - message_delta.usage.cache_creation_input_tokens = - details.cache_creation_input_tokens + if creation > 0 then + message_delta.usage.cache_creation_input_tokens = creation end end table.insert(events, make_sse_event("message_delta", message_delta)) @@ -1000,19 +998,19 @@ local function openai_to_anthropic_sse(openai_chunk, state, tool_name_map) } if type(openai_chunk.usage) == "table" then - local details = openai_chunk.usage.prompt_tokens_details local prompt_tokens = openai_chunk.usage.prompt_tokens or 0 - local cached = 0 - if type(details) == "table" then - cached = details.cached_tokens or 0 - end + local cached = openai_chat.cache_read_tokens(openai_chunk.usage) + local creation = openai_chat.cache_creation_tokens(openai_chunk.usage) message_delta.usage = { - input_tokens = math_max(0, prompt_tokens - cached), + input_tokens = math_max(0, prompt_tokens - cached - creation), output_tokens = openai_chunk.usage.completion_tokens or 0, } if cached > 0 then message_delta.usage.cache_read_input_tokens = cached end + if creation > 0 then + message_delta.usage.cache_creation_input_tokens = creation + end end state.pending_message_delta = message_delta diff --git a/apisix/plugins/ai-protocols/openai-chat.lua b/apisix/plugins/ai-protocols/openai-chat.lua index a92b5164f534..8870b8d4c524 100644 --- a/apisix/plugins/ai-protocols/openai-chat.lua +++ b/apisix/plugins/ai-protocols/openai-chat.lua @@ -30,6 +30,54 @@ local ngx_time = ngx.time local _M = {} +-- prompt_tokens_details keys that carry cache-write tokens, in LiteLLM's +-- order, followed by the cache_creation_input_tokens shape some upstreams use. +local CACHE_CREATION_DETAIL_KEYS = { + "cache_creation_tokens", + "cache_write_tokens", + "cache_creation_input_tokens", +} + + +-- Shared by usage extraction and the Anthropic converter so that both +-- report the same cache counts for an OpenAI-compatible usage object. +function _M.cache_read_tokens(usage) + if type(usage) ~= "table" then + return 0 + end + local details = usage.prompt_tokens_details + if type(details) == "table" and type(details.cached_tokens) == "number" then + return details.cached_tokens + end + -- DeepSeek + if type(usage.prompt_cache_hit_tokens) == "number" then + return usage.prompt_cache_hit_tokens + end + return 0 +end + + +function _M.cache_creation_tokens(usage) + if type(usage) ~= "table" then + return 0 + end + local v = usage.cache_creation_input_tokens + if type(v) == "number" and v > 0 then + return v + end + local details = usage.prompt_tokens_details + if type(details) == "table" then + for _, key in ipairs(CACHE_CREATION_DETAIL_KEYS) do + v = details[key] + if type(v) == "number" and v > 0 then + return v + end + end + end + return 0 +end + + --- Detect whether the request body matches OpenAI Chat Completions format. -- Body-only detection: matches any request with a messages array. function _M.matches(body, ctx) @@ -103,16 +151,14 @@ function _M.parse_sse_event(event, ctx, state) -- Extract usage (null for non-final chunks; cjson decodes null as userdata) if type(data.usage) == "table" then local u = data.usage - local pd = type(u.prompt_tokens_details) == "table" and u.prompt_tokens_details local cd = type(u.completion_tokens_details) == "table" and u.completion_tokens_details result.type = "usage" result.usage = { prompt_tokens = u.prompt_tokens or 0, completion_tokens = u.completion_tokens or 0, total_tokens = u.total_tokens or 0, - cache_read_input_tokens = pd and pd.cached_tokens - or u.prompt_cache_hit_tokens or 0, - cache_creation_input_tokens = pd and pd.cache_creation_input_tokens or 0, + cache_read_input_tokens = _M.cache_read_tokens(u), + cache_creation_input_tokens = _M.cache_creation_tokens(u), reasoning_tokens = cd and cd.reasoning_tokens or 0, } result.raw_usage = u @@ -173,17 +219,14 @@ function _M.extract_usage(res_body) return nil, nil end local raw = res_body.usage - local pdetails = type(raw.prompt_tokens_details) == "table" and raw.prompt_tokens_details local cdetails = type(raw.completion_tokens_details) == "table" and raw.completion_tokens_details - -- OpenAI uses prompt_tokens_details.cached_tokens; DeepSeek uses prompt_cache_hit_tokens - local cache_read = pdetails and pdetails.cached_tokens or raw.prompt_cache_hit_tokens or 0 return { prompt_tokens = raw.prompt_tokens or 0, completion_tokens = raw.completion_tokens or 0, total_tokens = raw.total_tokens or (raw.prompt_tokens or 0) + (raw.completion_tokens or 0), - cache_read_input_tokens = cache_read, - cache_creation_input_tokens = pdetails and pdetails.cache_creation_input_tokens or 0, + cache_read_input_tokens = _M.cache_read_tokens(raw), + cache_creation_input_tokens = _M.cache_creation_tokens(raw), reasoning_tokens = cdetails and cdetails.reasoning_tokens or 0, }, raw end diff --git a/docs/en/latest/plugins/ai-proxy-multi.md b/docs/en/latest/plugins/ai-proxy-multi.md index 425771e3024a..2246fe8aa6e8 100644 --- a/docs/en/latest/plugins/ai-proxy-multi.md +++ b/docs/en/latest/plugins/ai-proxy-multi.md @@ -2641,9 +2641,9 @@ The following example demonstrates how you can log LLM request related informati * `request_type`: Type of request, where the value could be `traditional_http`, `ai_chat`, or `ai_stream`. * `llm_time_to_first_token`: Duration from request sending to the first token received from the LLM service, in milliseconds. For an error response from the LLM service, this is the duration until the error response was received. It stays `0` when the LLM service could not be reached at all. * `llm_model`: LLM model. -* `llm_prompt_tokens`: Number of tokens in the prompt. +* `llm_prompt_tokens`: Number of tokens in the prompt, including tokens read from and written to the prompt cache. For the Anthropic Messages API, this is `input_tokens + cache_read_input_tokens + cache_creation_input_tokens`, which matches the meaning of OpenAI's `prompt_tokens`. * `llm_completion_tokens`: Number of chat completion tokens in the response. -* `llm_total_tokens`: Total number of tokens used (prompt plus completion). +* `llm_total_tokens`: Total number of tokens used (prompt plus completion, including cache tokens). * `llm_cache_read_input_tokens`: Number of input tokens read from cache. * `llm_cache_creation_input_tokens`: Number of input tokens written to cache. * `llm_reasoning_tokens`: Number of reasoning tokens generated. diff --git a/docs/en/latest/plugins/ai-proxy.md b/docs/en/latest/plugins/ai-proxy.md index 5e24268e53fa..090d19254809 100644 --- a/docs/en/latest/plugins/ai-proxy.md +++ b/docs/en/latest/plugins/ai-proxy.md @@ -2100,9 +2100,9 @@ The following example demonstrates how you can log LLM request related informati * `request_type`: Type of request, where the value could be `traditional_http`, `ai_chat`, or `ai_stream`. * `llm_time_to_first_token`: Duration from request sending to the first token received from the LLM service, in milliseconds. For an error response from the LLM service, this is the duration until the error response was received. It stays `0` when the LLM service could not be reached at all. * `llm_model`: LLM model. -* `llm_prompt_tokens`: Number of tokens in the prompt. +* `llm_prompt_tokens`: Number of tokens in the prompt, including tokens read from and written to the prompt cache. For the Anthropic Messages API, this is `input_tokens + cache_read_input_tokens + cache_creation_input_tokens`, which matches the meaning of OpenAI's `prompt_tokens`. * `llm_completion_tokens`: Number of chat completion tokens in the response. -* `llm_total_tokens`: Total number of tokens used (prompt plus completion). +* `llm_total_tokens`: Total number of tokens used (prompt plus completion, including cache tokens). * `llm_cache_read_input_tokens`: Number of input tokens read from cache. * `llm_cache_creation_input_tokens`: Number of input tokens written to cache. * `llm_reasoning_tokens`: Number of reasoning tokens generated. diff --git a/docs/en/latest/plugins/ai-rate-limiting.md b/docs/en/latest/plugins/ai-rate-limiting.md index 76f758a8c9b1..e5bc274b7a25 100644 --- a/docs/en/latest/plugins/ai-rate-limiting.md +++ b/docs/en/latest/plugins/ai-rate-limiting.md @@ -47,7 +47,7 @@ The `ai-rate-limiting` Plugin enforces token-based rate limiting for requests se | limit | integer | False | | >0 | The maximum number of tokens allowed within a given time interval. At least one of `limit` and `instances.limit` should be configured. Required if `rules` is not configured. | | time_window | integer | False | | >0 | The time interval corresponding to the rate limiting `limit` in seconds. At least one of `time_window` and `instances.time_window` should be configured. Required if `rules` is not configured. | | show_limit_quota_header | boolean | False | true | | If true, includes rate limiting response headers. When `rules` is not set, the headers are `X-AI-RateLimit-Limit-*`, `X-AI-RateLimit-Remaining-*`, and `X-AI-RateLimit-Reset-*`, where `*` is the instance name. When `rules` is set, see `rules.header_prefix` for details. | -| limit_strategy | string | False | total_tokens | [`total_tokens`, `prompt_tokens`, `completion_tokens`, `expression`] | Type of token to apply rate limiting. `total_tokens` is the sum of `prompt_tokens` and `completion_tokens`. When set to `expression`, the `cost_expr` field is used to dynamically calculate token cost. | +| limit_strategy | string | False | total_tokens | [`total_tokens`, `prompt_tokens`, `completion_tokens`, `expression`] | Type of token to apply rate limiting. `total_tokens` is the sum of `prompt_tokens` and `completion_tokens`. `prompt_tokens` includes tokens read from and written to the prompt cache, for the Anthropic Messages API as well as OpenAI-compatible APIs. When set to `expression`, the `cost_expr` field is used to dynamically calculate token cost. | | cost_expr | string | False | | | Lua arithmetic expression for dynamic token cost calculation. Variables are injected from the LLM API raw usage response fields. Nested fields are referenced by joining the parent and child keys with `__`, e.g. `input_tokens_details__cached_tokens` for `input_tokens_details.cached_tokens`. Arrays are skipped. Missing variables default to 0. Only valid when `limit_strategy` is `expression`. Example: `input_tokens + cache_creation_input_tokens + output_tokens`. | | instances | array[object] | False | | | LLM instance rate limiting configurations. | | instances.name | string | True | | | Name of the LLM service instance. | diff --git a/docs/en/latest/plugins/prometheus.md b/docs/en/latest/plugins/prometheus.md index 2b183020ea6c..e0f16dad9ebf 100644 --- a/docs/en/latest/plugins/prometheus.md +++ b/docs/en/latest/plugins/prometheus.md @@ -333,6 +333,8 @@ APISIX increments this gauge when an LLM upstream attempt starts and decrements ### Labels for `apisix_llm_prompt_tokens` +Prompt tokens include tokens read from and written to the prompt cache. For the Anthropic Messages API, this is `input_tokens + cache_read_input_tokens + cache_creation_input_tokens`, which matches the meaning of OpenAI's `prompt_tokens`. + | Name | Description | | ---------- | ----------------------------------------------------------------------------------------------------------------------------- | | route_id | ID of the Route that the metric corresponds to when `prefer_name` is `false` (default), and name of the Route when `prefer_name` is `true`. Defaults to an empty string if no Route is matched. | diff --git a/docs/zh/latest/plugins/ai-proxy-multi.md b/docs/zh/latest/plugins/ai-proxy-multi.md index dd50b025930c..db6e43e198a8 100644 --- a/docs/zh/latest/plugins/ai-proxy-multi.md +++ b/docs/zh/latest/plugins/ai-proxy-multi.md @@ -2732,9 +2732,9 @@ curl "http://127.0.0.1:9080/anything" -X POST \ * `request_type`：请求类型，值可能是 `traditional_http`、`ai_chat` 或 `ai_stream`。 * `llm_time_to_first_token`：从发送请求到从 LLM 服务接收第一个令牌的持续时间（毫秒）。当 LLM 服务返回错误响应时，该值为收到错误响应为止的持续时间；当完全无法连接到 LLM 服务时，该值保持为 `0`。 * `llm_model`：LLM 模型。 -* `llm_prompt_tokens`：提示中的令牌数量。 +* `llm_prompt_tokens`：提示中的令牌数量，包含从缓存读取和写入缓存的令牌。对于 Anthropic Messages API，其值为 `input_tokens + cache_read_input_tokens + cache_creation_input_tokens`，与 OpenAI 的 `prompt_tokens` 含义一致。 * `llm_completion_tokens`：响应中的聊天完成令牌数量。 -* `llm_total_tokens`：使用的总令牌数（提示加完成）。 +* `llm_total_tokens`：使用的总令牌数（提示加完成，包含缓存令牌）。 * `llm_cache_read_input_tokens`：从缓存读取的输入令牌数量。 * `llm_cache_creation_input_tokens`：写入缓存的输入令牌数量。 * `llm_reasoning_tokens`：生成的推理令牌数量。 diff --git a/docs/zh/latest/plugins/ai-proxy.md b/docs/zh/latest/plugins/ai-proxy.md index bdabdd300a1d..55a1f4811ca6 100644 --- a/docs/zh/latest/plugins/ai-proxy.md +++ b/docs/zh/latest/plugins/ai-proxy.md @@ -2080,9 +2080,9 @@ curl "http://127.0.0.1:9080/anything" -X POST \ * `request_type`：请求类型，值可能是 `traditional_http`、`ai_chat` 或 `ai_stream`。 * `llm_time_to_first_token`：从发送请求到从 LLM 服务接收第一个令牌的持续时间（毫秒）。当 LLM 服务返回错误响应时，该值为收到错误响应为止的持续时间；当完全无法连接到 LLM 服务时，该值保持为 `0`。 * `llm_model`：LLM 模型。 -* `llm_prompt_tokens`：提示中的令牌数量。 +* `llm_prompt_tokens`：提示中的令牌数量，包含从缓存读取和写入缓存的令牌。对于 Anthropic Messages API，其值为 `input_tokens + cache_read_input_tokens + cache_creation_input_tokens`，与 OpenAI 的 `prompt_tokens` 含义一致。 * `llm_completion_tokens`：响应中的聊天完成令牌数量。 -* `llm_total_tokens`：使用的总令牌数（提示加完成）。 +* `llm_total_tokens`：使用的总令牌数（提示加完成，包含缓存令牌）。 * `llm_cache_read_input_tokens`：从缓存读取的输入令牌数量。 * `llm_cache_creation_input_tokens`：写入缓存的输入令牌数量。 * `llm_reasoning_tokens`：生成的推理令牌数量。 diff --git a/docs/zh/latest/plugins/ai-rate-limiting.md b/docs/zh/latest/plugins/ai-rate-limiting.md index 7a2a728a9bf3..da72daf65e37 100644 --- a/docs/zh/latest/plugins/ai-rate-limiting.md +++ b/docs/zh/latest/plugins/ai-rate-limiting.md @@ -47,7 +47,7 @@ import TabItem from '@theme/TabItem'; | limit | integer | False | | >0 | 在给定时间间隔内允许的最大令牌数。`limit` 和 `instances.limit` 中至少应配置一个。如果未配置 `rules`，则为必填项。 | | time_window | integer | False | | >0 | 与速率限制 `limit` 对应的时间间隔（秒）。`time_window` 和 `instances.time_window` 中至少应配置一个。如果未配置 `rules`，则为必填项。 | | show_limit_quota_header | boolean | False | true | | 如果为 true，则在响应中包含速率限制头部。当未设置 `rules` 时，头部为 `X-AI-RateLimit-Limit-*`、`X-AI-RateLimit-Remaining-*` 和 `X-AI-RateLimit-Reset-*`，其中 `*` 是实例名称。当设置了 `rules` 时，详见 `rules.header_prefix`。 | -| limit_strategy | string | False | total_tokens | [`total_tokens`, `prompt_tokens`, `completion_tokens`, `expression`] | 应用速率限制的令牌类型。`total_tokens` 是 `prompt_tokens` 和 `completion_tokens` 的总和。当设置为 `expression` 时，使用 `cost_expr` 字段动态计算令牌消耗。 | +| limit_strategy | string | False | total_tokens | [`total_tokens`, `prompt_tokens`, `completion_tokens`, `expression`] | 应用速率限制的令牌类型。`total_tokens` 是 `prompt_tokens` 和 `completion_tokens` 的总和。`prompt_tokens` 包含从缓存读取和写入缓存的令牌，Anthropic Messages API 与 OpenAI 兼容 API 均如此。当设置为 `expression` 时，使用 `cost_expr` 字段动态计算令牌消耗。 | | cost_expr | string | False | | | 用于动态计算令牌消耗的 Lua 算术表达式。变量从 LLM API 原始使用量响应字段注入。嵌套字段通过用 `__` 连接父字段名和子字段名来引用，例如用 `input_tokens_details__cached_tokens` 引用 `input_tokens_details.cached_tokens`。数组会被跳过。缺失的变量默认为 0。仅在 `limit_strategy` 为 `expression` 时有效。示例：`input_tokens + cache_creation_input_tokens + output_tokens`。 | | instances | array[object] | False | | | LLM 实例速率限制配置。 | | instances.name | string | True | | | LLM 服务实例的名称。 | diff --git a/docs/zh/latest/plugins/prometheus.md b/docs/zh/latest/plugins/prometheus.md index ce772bfa7fb5..13128a44ac80 100644 --- a/docs/zh/latest/plugins/prometheus.md +++ b/docs/zh/latest/plugins/prometheus.md @@ -321,6 +321,8 @@ APISIX 在开始 LLM 上游尝试时递增该 gauge，并在请求的日志阶 ### `apisix_llm_prompt_tokens` 的标签 +prompt token 数包含从缓存读取和写入缓存的 token 数。对于 Anthropic Messages API，其值为 `input_tokens + cache_read_input_tokens + cache_creation_input_tokens`，与 OpenAI 的 `prompt_tokens` 含义一致。 + | 名称 | 描述 | | ---------- | ----------------------------------------------------------------------------------------------------------------------------- | | route_id | 指标对应的路由 ID；`prefer_name` 为 `true` 时保存路由名称。如果请求未匹配任何路由，则默认为空字符串。 | diff --git a/t/fixtures/anthropic/messages-streaming-with-cache-in-delta.sse b/t/fixtures/anthropic/messages-streaming-with-cache-in-delta.sse new file mode 100644 index 000000000000..91c3b719b69b --- /dev/null +++ b/t/fixtures/anthropic/messages-streaming-with-cache-in-delta.sse @@ -0,0 +1,18 @@ +event: message_start +data: {"type":"message_start","message":{"id":"msg_test789","type":"message","role":"assistant","model":"{{model}}","content":[],"usage":{"input_tokens":50,"output_tokens":1,"cache_read_input_tokens":0}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello from Claude!"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":30,"cache_read_input_tokens":200,"cache_creation_input_tokens":100}} + +event: message_stop +data: {} + diff --git a/t/fixtures/openai/chat-streaming-cache-creation-tokens.sse b/t/fixtures/openai/chat-streaming-cache-creation-tokens.sse new file mode 100644 index 000000000000..768938e5d347 --- /dev/null +++ b/t/fixtures/openai/chat-streaming-cache-creation-tokens.sse @@ -0,0 +1,6 @@ +data: {"id":"chatcmpl-cache1","object":"chat.completion.chunk","choices":[{"delta":{"role":"assistant","content":"Hi"},"index":0,"finish_reason":null}],"usage":null} + +data: {"id":"chatcmpl-cache1","object":"chat.completion.chunk","choices":[{"delta":{},"index":0,"finish_reason":"stop"}],"usage":{"prompt_tokens":30,"completion_tokens":15,"total_tokens":45,"prompt_tokens_details":{"cached_tokens":10,"cache_creation_tokens":5}}} + +data: [DONE] + diff --git a/t/fixtures/openai/chat-streaming-cache-write-tokens.sse b/t/fixtures/openai/chat-streaming-cache-write-tokens.sse new file mode 100644 index 000000000000..e597651d6c73 --- /dev/null +++ b/t/fixtures/openai/chat-streaming-cache-write-tokens.sse @@ -0,0 +1,6 @@ +data: {"id":"chatcmpl-cache1","object":"chat.completion.chunk","choices":[{"delta":{"role":"assistant","content":"Hi"},"index":0,"finish_reason":null}],"usage":null} + +data: {"id":"chatcmpl-cache1","object":"chat.completion.chunk","choices":[{"delta":{},"index":0,"finish_reason":"stop"}],"usage":{"prompt_tokens":30,"completion_tokens":15,"total_tokens":45,"prompt_tokens_details":{"cached_tokens":10,"cache_write_tokens":5}}} + +data: [DONE] + diff --git a/t/fixtures/openai/chat-streaming-cache-write-top-level.sse b/t/fixtures/openai/chat-streaming-cache-write-top-level.sse new file mode 100644 index 000000000000..ea7161e9f330 --- /dev/null +++ b/t/fixtures/openai/chat-streaming-cache-write-top-level.sse @@ -0,0 +1,6 @@ +data: {"id":"chatcmpl-cache1","object":"chat.completion.chunk","choices":[{"delta":{"role":"assistant","content":"Hi"},"index":0,"finish_reason":null}],"usage":null} + +data: {"id":"chatcmpl-cache1","object":"chat.completion.chunk","choices":[{"delta":{},"index":0,"finish_reason":"stop"}],"usage":{"prompt_tokens":30,"completion_tokens":15,"total_tokens":45,"cache_creation_input_tokens":5,"prompt_tokens_details":{"cached_tokens":10}}} + +data: [DONE] + diff --git a/t/fixtures/openai/chat-streaming-deepseek-cache.sse b/t/fixtures/openai/chat-streaming-deepseek-cache.sse new file mode 100644 index 000000000000..488d3e0315f1 --- /dev/null +++ b/t/fixtures/openai/chat-streaming-deepseek-cache.sse @@ -0,0 +1,6 @@ +data: {"id":"chatcmpl-cache1","object":"chat.completion.chunk","choices":[{"delta":{"role":"assistant","content":"Hi"},"index":0,"finish_reason":null}],"usage":null} + +data: {"id":"chatcmpl-cache1","object":"chat.completion.chunk","choices":[{"delta":{},"index":0,"finish_reason":"stop"}],"usage":{"prompt_tokens":30,"completion_tokens":15,"total_tokens":45,"prompt_cache_hit_tokens":10,"prompt_cache_miss_tokens":20}} + +data: [DONE] + diff --git a/t/plugin/ai-proxy-anthropic.t b/t/plugin/ai-proxy-anthropic.t index ade9dad62f18..db575e6a65ae 100644 --- a/t/plugin/ai-proxy-anthropic.t +++ b/t/plugin/ai-proxy-anthropic.t @@ -181,7 +181,7 @@ qr/(?s)(?=.*"type":"thinking")(?=.*"thinking":"Let me think step by step)(?=.*"s -=== TEST 8: cached_tokens deducted from input_tokens +=== TEST 8: cached_tokens and cache_creation_input_tokens deducted from input_tokens --- request POST /v1/messages {"model":"claude-sonnet-4-20250514","max_tokens":1024,"messages":[{"role":"user","content":"test"}]} @@ -190,7 +190,7 @@ Content-Type: application/json X-AI-Fixture: openai/chat-with-reasoning.json --- error_code: 200 --- response_body_like eval -qr/(?s)(?=.*"input_tokens":20)(?=.*"cache_read_input_tokens":10)(?=.*"cache_creation_input_tokens":5)/ +qr/(?s)(?=.*"input_tokens":15)(?=.*"cache_read_input_tokens":10)(?=.*"cache_creation_input_tokens":5)/ --- no_error_log [error] @@ -1808,7 +1808,7 @@ qr/127\.0\.0\.1:1980 200 [\d.]+ \"\S+\" claude-3-5-sonnet-20241022 claude-3-5-so -=== TEST 50: Anthropic non-streaming writes cache tokens to access log +=== TEST 50: Anthropic non-streaming writes cache tokens to access log, prompt tokens include cache --- request POST /v1/messages {"messages":[{"role":"user","content":"Hello"}],"model":"claude-3-5-sonnet-20241022","max_tokens":1024} @@ -1816,11 +1816,11 @@ POST /v1/messages X-AI-Fixture: anthropic/messages-with-cache.json --- error_code: 200 --- access_log eval -qr/127\.0\.0\.1:1980 200 [\d.]+ \"\S+\" claude-3-5-sonnet-20241022 claude-3-5-sonnet-20241022 [\d.]+ 50 30 80 false false 0 \S* 200 100 0/ +qr/127\.0\.0\.1:1980 200 [\d.]+ \"\S+\" claude-3-5-sonnet-20241022 claude-3-5-sonnet-20241022 [\d.]+ 350 30 380 false false 0 \S* 200 100 0/ -=== TEST 51: Anthropic streaming writes cache tokens to access log +=== TEST 51: Anthropic streaming writes cache tokens to access log, prompt tokens include cache --- request POST /v1/messages {"messages":[{"role":"user","content":"Hello"}],"model":"claude-3-5-sonnet-20241022","max_tokens":1024,"stream":true} @@ -1828,7 +1828,7 @@ POST /v1/messages X-AI-Fixture: anthropic/messages-streaming-with-cache.sse --- error_code: 200 --- access_log eval -qr/127\.0\.0\.1:1980 200 [\d.]+ \"\S+\" claude-3-5-sonnet-20241022 claude-3-5-sonnet-20241022 [\d.]+ 50 30 80 true false 0 \S* 200 100 0/ +qr/127\.0\.0\.1:1980 200 [\d.]+ \"\S+\" claude-3-5-sonnet-20241022 claude-3-5-sonnet-20241022 [\d.]+ 350 30 380 true false 0 \S* 200 100 0/ @@ -2479,3 +2479,315 @@ OK OK --- no_error_log [error] + + + +=== TEST 65: set route for native Anthropic protocol with cache tokens in message_delta +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/v1/messages", + "plugins": { + "ai-proxy": { + "provider": "anthropic", + "auth": { + "header": { + "x-api-key": "test-key" + } + }, + "options": { + "model": "claude-3-5-sonnet-20241022" + }, + "override": { + "endpoint": "http://127.0.0.1:1980" + }, + "ssl_verify": false + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 66: Anthropic streaming picks up cache tokens reported only in message_delta +--- request +POST /v1/messages +{"messages":[{"role":"user","content":"Hello"}],"model":"claude-3-5-sonnet-20241022","max_tokens":1024,"stream":true} +--- more_headers +X-AI-Fixture: anthropic/messages-streaming-with-cache-in-delta.sse +--- error_code: 200 +--- access_log eval +qr/127\.0\.0\.1:1980 200 [\d.]+ \"\S+\" claude-3-5-sonnet-20241022 claude-3-5-sonnet-20241022 [\d.]+ 350 30 380 true false 0 \S* 200 100 0/ + + + +=== TEST 67: set route for Anthropic client with OpenAI-compatible upstream +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/v1/messages", + "plugins": { + "ai-proxy": { + "provider": "openai-compatible", + "auth": { + "header": { + "Authorization": "Bearer test-token" + } + }, + "options": { + "model": "gpt-4o" + }, + "override": { + "endpoint": "http://127.0.0.1:1980" + }, + "ssl_verify": false + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 68: converted non-streaming response logs the upstream prompt_tokens, cache included +--- request +POST /v1/messages +{"model":"claude-sonnet-4-20250514","max_tokens":1024,"messages":[{"role":"user","content":"test"}]} +--- more_headers +Content-Type: application/json +X-AI-Fixture: openai/chat-with-reasoning.json +--- error_code: 200 +--- response_body_like eval +qr/(?s)(?=.*"input_tokens":15)(?=.*"cache_read_input_tokens":10)(?=.*"cache_creation_input_tokens":5)/ +--- access_log eval +qr/127\.0\.0\.1:1980 200 [\d.]+ \"\S+\" \S+ \S+ [\d.]+ 30 15 45 false false 0 \S* 10 5 0/ + + + +=== TEST 69: converted streaming response deducts cache tokens from input_tokens and logs the upstream prompt_tokens +--- request +POST /v1/messages +{"model":"claude-sonnet-4-20250514","max_tokens":1024,"stream":true,"messages":[{"role":"user","content":"test"}]} +--- more_headers +Content-Type: application/json +X-AI-Fixture: openai/chat-streaming-with-cache.sse +--- error_code: 200 +--- response_body_like eval +qr/(?s)(?=.*"input_tokens":15)(?=.*"cache_read_input_tokens":10)(?=.*"cache_creation_input_tokens":5)/ +--- access_log eval +qr/127\.0\.0\.1:1980 200 [\d.]+ \"\S+\" \S+ \S+ [\d.]+ 30 15 45 true false 0 \S* 10 5 \d+/ + + + +=== TEST 70: cache-creation tokens are read from the LiteLLM usage field shapes +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local converter = require("apisix.plugins.ai-protocols.converters.anthropic-messages-to-openai-chat") + + local function convert(usage) + local res = converter.convert_response({ + id = "x", model = "m", + choices = {{ message = { role = "assistant", content = "hi" }, + finish_reason = "stop" }}, + usage = usage, + }, { var = {} }) + return res.usage + end + + -- top-level usage.cache_creation_input_tokens + local u = convert({ prompt_tokens = 100, completion_tokens = 10, + cache_creation_input_tokens = 30, + prompt_tokens_details = { cached_tokens = 20 } }) + assert(u.input_tokens == 50, "top-level input: " .. u.input_tokens) + assert(u.cache_creation_input_tokens == 30, "top-level creation") + + -- prompt_tokens_details.cache_creation_tokens, no cached_tokens + u = convert({ prompt_tokens = 100, completion_tokens = 10, + prompt_tokens_details = { cache_creation_tokens = 30 } }) + assert(u.input_tokens == 70, "cache_creation_tokens input: " .. u.input_tokens) + assert(u.cache_creation_input_tokens == 30, "cache_creation_tokens creation") + + -- prompt_tokens_details.cache_write_tokens + u = convert({ prompt_tokens = 100, completion_tokens = 10, + prompt_tokens_details = { cached_tokens = 20, + cache_write_tokens = 30 } }) + assert(u.input_tokens == 50, "cache_write_tokens input: " .. u.input_tokens) + assert(u.cache_creation_input_tokens == 30, "cache_write_tokens creation") + + -- streaming: usage on the finish_reason chunk + local state = { is_first = true } + converter.convert_sse_events({ + type = "data", + data = { id = "x", model = "m", choices = {{ delta = { content = "hi" } }} }, + }, {}, state) + converter.convert_sse_events({ + type = "data", + data = { + choices = {{ delta = {}, finish_reason = "stop" }}, + usage = { prompt_tokens = 100, completion_tokens = 10, + prompt_tokens_details = { cached_tokens = 20, + cache_write_tokens = 30 } }, + }, + }, {}, state) + -- the deferred message_delta is flushed by the next chunk + local events = converter.convert_sse_events({ + type = "data", + data = { choices = {} }, + }, {}, state) + local msg_delta + for _, e in ipairs(events) do + if e.type == "message_delta" then + msg_delta = core.json.decode(e.data) + end + end + assert(msg_delta, "message_delta emitted") + assert(msg_delta.usage.input_tokens == 50, + "stream input: " .. msg_delta.usage.input_tokens) + assert(msg_delta.usage.cache_creation_input_tokens == 30, "stream creation") + + ngx.say("OK") + } + } +--- response_body +OK +--- no_error_log +[error] + + + +=== TEST 71: converted streaming logs cache-creation tokens from top-level usage.cache_creation_input_tokens +--- request +POST /v1/messages +{"model":"claude-sonnet-4-20250514","max_tokens":1024,"stream":true,"messages":[{"role":"user","content":"test"}]} +--- more_headers +Content-Type: application/json +X-AI-Fixture: openai/chat-streaming-cache-write-top-level.sse +--- error_code: 200 +--- response_body_like eval +qr/(?s)(?=.*"input_tokens":15)(?=.*"cache_read_input_tokens":10)(?=.*"cache_creation_input_tokens":5)/ +--- access_log eval +qr/127\.0\.0\.1:1980 200 [\d.]+ \"\S+\" \S+ \S+ [\d.]+ 30 15 45 true false 0 \S* 10 5 \d+/ + + + +=== TEST 72: converted streaming logs cache-creation tokens from prompt_tokens_details.cache_write_tokens +--- request +POST /v1/messages +{"model":"claude-sonnet-4-20250514","max_tokens":1024,"stream":true,"messages":[{"role":"user","content":"test"}]} +--- more_headers +Content-Type: application/json +X-AI-Fixture: openai/chat-streaming-cache-write-tokens.sse +--- error_code: 200 +--- response_body_like eval +qr/(?s)(?=.*"input_tokens":15)(?=.*"cache_read_input_tokens":10)(?=.*"cache_creation_input_tokens":5)/ +--- access_log eval +qr/127\.0\.0\.1:1980 200 [\d.]+ \"\S+\" \S+ \S+ [\d.]+ 30 15 45 true false 0 \S* 10 5 \d+/ + + + +=== TEST 73: converted streaming logs cache-creation tokens from prompt_tokens_details.cache_creation_tokens +--- request +POST /v1/messages +{"model":"claude-sonnet-4-20250514","max_tokens":1024,"stream":true,"messages":[{"role":"user","content":"test"}]} +--- more_headers +Content-Type: application/json +X-AI-Fixture: openai/chat-streaming-cache-creation-tokens.sse +--- error_code: 200 +--- response_body_like eval +qr/(?s)(?=.*"input_tokens":15)(?=.*"cache_read_input_tokens":10)(?=.*"cache_creation_input_tokens":5)/ +--- access_log eval +qr/127\.0\.0\.1:1980 200 [\d.]+ \"\S+\" \S+ \S+ [\d.]+ 30 15 45 true false 0 \S* 10 5 \d+/ + + + +=== TEST 74: converted streaming reads DeepSeek prompt_cache_hit_tokens as cache reads +--- request +POST /v1/messages +{"model":"claude-sonnet-4-20250514","max_tokens":1024,"stream":true,"messages":[{"role":"user","content":"test"}]} +--- more_headers +Content-Type: application/json +X-AI-Fixture: openai/chat-streaming-deepseek-cache.sse +--- error_code: 200 +--- response_body_like eval +qr/(?s)(?=.*"input_tokens":20)(?=.*"cache_read_input_tokens":10)(?!.*"cache_creation_input_tokens")/ +--- access_log eval +qr/127\.0\.0\.1:1980 200 [\d.]+ \"\S+\" \S+ \S+ [\d.]+ 30 15 45 true false 0 \S* 10 0 \d+/ + + + +=== TEST 75: OpenAI usage extraction reads every supported cache field shape +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local openai_chat = require("apisix.plugins.ai-protocols.openai-chat") + + local cases = { + { { prompt_tokens = 30, cache_creation_input_tokens = 5, + prompt_tokens_details = { cached_tokens = 10 } }, 10, 5 }, + { { prompt_tokens = 30, + prompt_tokens_details = { cached_tokens = 10, cache_write_tokens = 5 } }, + 10, 5 }, + { { prompt_tokens = 30, + prompt_tokens_details = { cached_tokens = 10, cache_creation_tokens = 5 } }, + 10, 5 }, + { { prompt_tokens = 30, + prompt_tokens_details = { cached_tokens = 10, + cache_creation_input_tokens = 5 } }, + 10, 5 }, + { { prompt_tokens = 30, prompt_cache_hit_tokens = 10 }, 10, 0 }, + { { prompt_tokens = 30 }, 0, 0 }, + } + for i, c in ipairs(cases) do + local raw, read, creation = c[1], c[2], c[3] + raw.completion_tokens = 15 + + local usage = openai_chat.extract_usage({ usage = raw }) + assert(usage.cache_read_input_tokens == read, "non-stream read " .. i) + assert(usage.cache_creation_input_tokens == creation, + "non-stream creation " .. i) + + local res = openai_chat.parse_sse_event({ + type = "message", + data = core.json.encode({ choices = {}, usage = raw }), + }, {}, {}) + assert(res.usage.cache_read_input_tokens == read, "stream read " .. i) + assert(res.usage.cache_creation_input_tokens == creation, + "stream creation " .. i) + assert(res.raw_usage.prompt_tokens == 30, "raw usage " .. i) + end + + ngx.say("OK") + } + } +--- response_body +OK +--- no_error_log +[error] diff --git a/t/plugin/ai-rate-limiting-expression.t b/t/plugin/ai-rate-limiting-expression.t index 3598ce0acafc..9ec58d8927c6 100644 --- a/t/plugin/ai-rate-limiting-expression.t +++ b/t/plugin/ai-rate-limiting-expression.t @@ -1027,3 +1027,73 @@ X-AI-Fixture: openai/chat-usage-deep.json ] --- no_error_log [error] + + + +=== TEST 29: set route with expression on Anthropic client converted to OpenAI upstream +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/v1/messages", + "plugins": { + "ai-proxy": { + "provider": "openai", + "auth": { + "header": { + "Authorization": "Bearer token" + } + }, + "options": { + "model": "gpt-4o-mini" + }, + "override": { + "endpoint": "http://127.0.0.1:1980" + }, + "ssl_verify": false + }, + "ai-rate-limiting": { + "limit": 500, + "time_window": 60, + "limit_strategy": "expression", + "cost_expr": "input_tokens + cache_read_input_tokens + cache_creation_input_tokens" + } + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "canbeanything.com": 1 + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 30: converted non-streaming usage is in Anthropic shape, cache tokens not double-counted - cost = 15 + 10 + 5 = 30 (upstream prompt_tokens) per request +--- pipelined_requests eval +[ + "POST /v1/messages\n" . '{"model":"claude-sonnet-4-20250514","max_tokens":1024,"messages":[{"role":"user","content":"Hello"}]}', + "POST /v1/messages\n" . '{"model":"claude-sonnet-4-20250514","max_tokens":1024,"messages":[{"role":"user","content":"Hello"}]}', +] +--- more_headers +X-AI-Fixture: openai/chat-with-reasoning.json +--- response_headers_like eval +[ + "X-AI-RateLimit-Remaining-ai-proxy-openai: 500", + "X-AI-RateLimit-Remaining-ai-proxy-openai: 470", +] +--- no_error_log +[error] diff --git a/t/plugin/ai-rate-limiting.t b/t/plugin/ai-rate-limiting.t index c5ea8877bc36..370c96b304f1 100644 --- a/t/plugin/ai-rate-limiting.t +++ b/t/plugin/ai-rate-limiting.t @@ -1882,3 +1882,64 @@ passed passed --- error_log failed to get token usage for llm service + + + +=== TEST 43: set route with total_tokens strategy on native Anthropic +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/anthropic/v1/messages", + "plugins": { + "ai-proxy": { + "provider": "anthropic", + "auth": { + "header": { + "x-api-key": "test-key", + "anthropic-version": "2023-06-01" + } + }, + "options": { + "model": "claude-sonnet-4-20250514" + }, + "override": { + "endpoint": "http://127.0.0.1:1980" + }, + "ssl_verify": false + }, + "ai-rate-limiting": { + "limit": 300, + "time_window": 60, + "limit_strategy": "total_tokens" + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 44: Anthropic total_tokens includes cache tokens - 50 + 200 + 100 + 30 = 380 exceeds limit=300, second rejected +--- more_headers +X-AI-Fixture: anthropic/messages-with-cache.json +--- pipelined_requests eval +[ + "POST /anthropic/v1/messages\n" . '{"model":"claude-sonnet-4-20250514","max_tokens":1024,"messages":[{"role":"user","content":"Hello"}]}', + "POST /anthropic/v1/messages\n" . '{"model":"claude-sonnet-4-20250514","max_tokens":1024,"messages":[{"role":"user","content":"Hello"}]}', +] +--- error_code eval +[200, 503] +--- no_error_log +[error]