fix(ai-proxy): include cache tokens in Anthropic prompt usage and read them from message_delta

apisix/plugins/ai-protocols/anthropic-messages.lua

@@ -29,10 +29,51 @@ local sse = require("apisix.plugins.ai-transport.sse")
 local table = table
 local type = type
 local ipairs = ipairs
+local pairs = pairs
 
 local _M = {}
 
 
+-- Anthropic reports cache tokens alongside input_tokens rather than inside it,
+-- so add them back to get a prompt total with the same meaning as OpenAI's
+-- prompt_tokens. Mirrors LiteLLM's AnthropicConfig.calculate_usage.
+local function to_usage(raw)
+    local cache_read = raw.cache_read_input_tokens or 0
+    local cache_creation = raw.cache_creation_input_tokens or 0
+    local prompt = (raw.input_tokens or 0) + cache_read + cache_creation
+    local completion = raw.output_tokens or 0
+    return {
+        prompt_tokens = prompt,
+        completion_tokens = completion,
+        total_tokens = prompt + completion,
+        cache_read_input_tokens = cache_read,
+        cache_creation_input_tokens = cache_creation,
+        reasoning_tokens = 0,
+    }
+end
+
+
+-- message_delta usage is cumulative but may omit fields reported at
+-- message_start (and some providers only report cache tokens there), so
+-- merge the raw fields across events before computing usage. A later event
+-- overwrites a field rather than adding to it, because the values are
+-- cumulative: summing them would double-count.
+local function merge_usage_fields(state, raw)
+    if type(state) ~= "table" then
+        return raw
+    end
+    local acc = state.anthropic_usage
+    if not acc then
+        acc = {}
+        state.anthropic_usage = acc
+    end
+    for k, v in pairs(raw) do
+        acc[k] = v
+    end
+    return acc
+end
+
+
 --- Detect whether the request matches the Anthropic Messages API format.
 -- Uses URI suffix (/v1/messages) and body (valid JSON table).
 function _M.matches(body, ctx)
@@ -83,12 +124,7 @@ function _M.parse_sse_event(event, ctx, state)
         if type(data.usage) == "table" then
             return {
                 type = "usage",
-                usage = {
-                    prompt_tokens = data.usage.input_tokens or 0,
-                    completion_tokens = data.usage.output_tokens or 0,
-                    total_tokens = (data.usage.input_tokens or 0)
-                        + (data.usage.output_tokens or 0),
-                },
+                usage = to_usage(merge_usage_fields(state, data.usage)),
                 raw_usage = data.usage,
             }
         end
@@ -106,14 +142,7 @@ function _M.parse_sse_event(event, ctx, state)
             local usage = data.message.usage
             return {
                 type = "usage",
-                usage = {
-                    prompt_tokens = usage.input_tokens or 0,
-                    completion_tokens = usage.output_tokens or 0,
-                    total_tokens = (usage.input_tokens or 0) + (usage.output_tokens or 0),
-                    cache_read_input_tokens = usage.cache_read_input_tokens or 0,
-                    cache_creation_input_tokens = usage.cache_creation_input_tokens or 0,
-                    reasoning_tokens = 0,
-                },
+                usage = to_usage(merge_usage_fields(state, usage)),
                 raw_usage = usage,
             }
         end
@@ -174,16 +203,7 @@ function _M.extract_usage(res_body)
         return nil, nil
     end
     local raw = res_body.usage
-    local prompt = raw.input_tokens or 0
-    local completion = raw.output_tokens or 0
-    return {
-        prompt_tokens = prompt,
-        completion_tokens = completion,
-        total_tokens = prompt + completion,
-        cache_read_input_tokens = raw.cache_read_input_tokens or 0,
-        cache_creation_input_tokens = raw.cache_creation_input_tokens or 0,
-        reasoning_tokens = 0,
-    }, raw
+    return to_usage(raw), raw
 end
 
 

apisix/plugins/ai-protocols/converters/anthropic-messages-to-openai-chat.lua

@@ -25,6 +25,7 @@
 -- fields never reach the upstream provider.
 
 local core = require("apisix.core")
+local openai_chat = require("apisix.plugins.ai-protocols.openai-chat")
 local resty_sha256 = require("resty.sha256")
 local to_hex = require("resty.string").to_hex
 local table = table
@@ -787,15 +788,16 @@ function _M.convert_response(res_body, ctx)
 
         usage.output_tokens = completion_tokens
 
-        if type(details) == "table" then
-            local cached = details.cached_tokens or 0
-            usage.input_tokens = math_max(0, prompt_tokens - cached)
+        -- Anthropic input_tokens excludes both cache reads and cache writes,
+        -- while OpenAI prompt_tokens includes them (same as LiteLLM).
+        local cached = openai_chat.cache_read_tokens(res_body.usage)
+        local creation = openai_chat.cache_creation_tokens(res_body.usage)
+        usage.input_tokens = math_max(0, prompt_tokens - cached - creation)
+        if type(details) == "table" or cached > 0 then
             usage.cache_read_input_tokens = cached
-            if details.cache_creation_input_tokens then
-                usage.cache_creation_input_tokens = details.cache_creation_input_tokens
-            end
-        else
-            usage.input_tokens = prompt_tokens
+        end
+        if creation > 0 then
+            usage.cache_creation_input_tokens = creation
         end
     end
 
@@ -830,22 +832,18 @@ local function openai_to_anthropic_sse(openai_chunk, state, tool_name_map)
         if state.pending_stop then
             local message_delta = state.pending_message_delta
             if type(openai_chunk.usage) == "table" and not message_delta.usage then
-                local details = openai_chunk.usage.prompt_tokens_details
                 local prompt_tokens = openai_chunk.usage.prompt_tokens or 0
-                local cached = 0
-                if type(details) == "table" then
-                    cached = details.cached_tokens or 0
-                end
+                local cached = openai_chat.cache_read_tokens(openai_chunk.usage)
+                local creation = openai_chat.cache_creation_tokens(openai_chunk.usage)
                 message_delta.usage = {
-                    input_tokens  = math_max(0, prompt_tokens - cached),
+                    input_tokens  = math_max(0, prompt_tokens - cached - creation),
                     output_tokens = openai_chunk.usage.completion_tokens or 0,
                 }
                 if cached > 0 then
                     message_delta.usage.cache_read_input_tokens = cached
                 end
-                if type(details) == "table" and details.cache_creation_input_tokens then
-                    message_delta.usage.cache_creation_input_tokens =
-                        details.cache_creation_input_tokens
+                if creation > 0 then
+                    message_delta.usage.cache_creation_input_tokens = creation
                 end
             end
             table.insert(events, make_sse_event("message_delta", message_delta))
@@ -1000,19 +998,19 @@ local function openai_to_anthropic_sse(openai_chunk, state, tool_name_map)
         }
 
         if type(openai_chunk.usage) == "table" then
-            local details = openai_chunk.usage.prompt_tokens_details
             local prompt_tokens = openai_chunk.usage.prompt_tokens or 0
-            local cached = 0
-            if type(details) == "table" then
-                cached = details.cached_tokens or 0
-            end
+            local cached = openai_chat.cache_read_tokens(openai_chunk.usage)
+            local creation = openai_chat.cache_creation_tokens(openai_chunk.usage)
             message_delta.usage = {
-                input_tokens  = math_max(0, prompt_tokens - cached),
+                input_tokens  = math_max(0, prompt_tokens - cached - creation),
                 output_tokens = openai_chunk.usage.completion_tokens or 0,
             }
             if cached > 0 then
                 message_delta.usage.cache_read_input_tokens = cached
             end
+            if creation > 0 then
+                message_delta.usage.cache_creation_input_tokens = creation
+            end
         end
 
         state.pending_message_delta = message_delta

apisix/plugins/ai-protocols/openai-chat.lua

@@ -30,6 +30,54 @@ local ngx_time = ngx.time
 local _M = {}
 
 
+-- prompt_tokens_details keys that carry cache-write tokens, in LiteLLM's
+-- order, followed by the cache_creation_input_tokens shape some upstreams use.
+local CACHE_CREATION_DETAIL_KEYS = {
+    "cache_creation_tokens",
+    "cache_write_tokens",
+    "cache_creation_input_tokens",
+}
+
+
+-- Shared by usage extraction and the Anthropic converter so that both
+-- report the same cache counts for an OpenAI-compatible usage object.
+function _M.cache_read_tokens(usage)
+    if type(usage) ~= "table" then
+        return 0
+    end
+    local details = usage.prompt_tokens_details
+    if type(details) == "table" and type(details.cached_tokens) == "number" then
+        return details.cached_tokens
+    end
+    -- DeepSeek
+    if type(usage.prompt_cache_hit_tokens) == "number" then
+        return usage.prompt_cache_hit_tokens
+    end
+    return 0
+end
+
+
+function _M.cache_creation_tokens(usage)
+    if type(usage) ~= "table" then
+        return 0
+    end
+    local v = usage.cache_creation_input_tokens
+    if type(v) == "number" and v > 0 then
+        return v
+    end
+    local details = usage.prompt_tokens_details
+    if type(details) == "table" then
+        for _, key in ipairs(CACHE_CREATION_DETAIL_KEYS) do
+            v = details[key]
+            if type(v) == "number" and v > 0 then
+                return v
+            end
+        end
+    end
+    return 0
+end
+
+
 --- Detect whether the request body matches OpenAI Chat Completions format.
 -- Body-only detection: matches any request with a messages array.
 function _M.matches(body, ctx)
@@ -103,16 +151,14 @@ function _M.parse_sse_event(event, ctx, state)
         -- Extract usage (null for non-final chunks; cjson decodes null as userdata)
         if type(data.usage) == "table" then
             local u = data.usage
-            local pd = type(u.prompt_tokens_details) == "table" and u.prompt_tokens_details
             local cd = type(u.completion_tokens_details) == "table" and u.completion_tokens_details
             result.type = "usage"
             result.usage = {
                 prompt_tokens = u.prompt_tokens or 0,
                 completion_tokens = u.completion_tokens or 0,
                 total_tokens = u.total_tokens or 0,
-                cache_read_input_tokens = pd and pd.cached_tokens
-                                          or u.prompt_cache_hit_tokens or 0,
-                cache_creation_input_tokens = pd and pd.cache_creation_input_tokens or 0,
+                cache_read_input_tokens = _M.cache_read_tokens(u),
+                cache_creation_input_tokens = _M.cache_creation_tokens(u),
                 reasoning_tokens = cd and cd.reasoning_tokens or 0,
             }
             result.raw_usage = u
@@ -173,17 +219,14 @@ function _M.extract_usage(res_body)
         return nil, nil
     end
     local raw = res_body.usage
-    local pdetails = type(raw.prompt_tokens_details) == "table" and raw.prompt_tokens_details
     local cdetails = type(raw.completion_tokens_details) == "table"
                      and raw.completion_tokens_details
-    -- OpenAI uses prompt_tokens_details.cached_tokens; DeepSeek uses prompt_cache_hit_tokens
-    local cache_read = pdetails and pdetails.cached_tokens or raw.prompt_cache_hit_tokens or 0
     return {
         prompt_tokens = raw.prompt_tokens or 0,
         completion_tokens = raw.completion_tokens or 0,
         total_tokens = raw.total_tokens or (raw.prompt_tokens or 0) + (raw.completion_tokens or 0),
-        cache_read_input_tokens = cache_read,
-        cache_creation_input_tokens = pdetails and pdetails.cache_creation_input_tokens or 0,
+        cache_read_input_tokens = _M.cache_read_tokens(raw),
+        cache_creation_input_tokens = _M.cache_creation_tokens(raw),
         reasoning_tokens = cdetails and cdetails.reasoning_tokens or 0,
     }, raw
 end

docs/en/latest/plugins/ai-proxy-multi.md

@@ -2641,9 +2641,9 @@ The following example demonstrates how you can log LLM request related informati
 * `request_type`: Type of request, where the value could be `traditional_http`, `ai_chat`, or `ai_stream`.
 * `llm_time_to_first_token`: Duration from request sending to the first token received from the LLM service, in milliseconds. For an error response from the LLM service, this is the duration until the error response was received. It stays `0` when the LLM service could not be reached at all.
 * `llm_model`: LLM model.
-* `llm_prompt_tokens`: Number of tokens in the prompt.
+* `llm_prompt_tokens`: Number of tokens in the prompt, including tokens read from and written to the prompt cache. For the Anthropic Messages API, this is `input_tokens + cache_read_input_tokens + cache_creation_input_tokens`, which matches the meaning of OpenAI's `prompt_tokens`.
 * `llm_completion_tokens`: Number of chat completion tokens in the response.
-* `llm_total_tokens`: Total number of tokens used (prompt plus completion).
+* `llm_total_tokens`: Total number of tokens used (prompt plus completion, including cache tokens).
 * `llm_cache_read_input_tokens`: Number of input tokens read from cache.
 * `llm_cache_creation_input_tokens`: Number of input tokens written to cache.
 * `llm_reasoning_tokens`: Number of reasoning tokens generated.

docs/en/latest/plugins/ai-proxy.md

@@ -2100,9 +2100,9 @@ The following example demonstrates how you can log LLM request related informati
 * `request_type`: Type of request, where the value could be `traditional_http`, `ai_chat`, or `ai_stream`.
 * `llm_time_to_first_token`: Duration from request sending to the first token received from the LLM service, in milliseconds. For an error response from the LLM service, this is the duration until the error response was received. It stays `0` when the LLM service could not be reached at all.
 * `llm_model`: LLM model.
-* `llm_prompt_tokens`: Number of tokens in the prompt.
+* `llm_prompt_tokens`: Number of tokens in the prompt, including tokens read from and written to the prompt cache. For the Anthropic Messages API, this is `input_tokens + cache_read_input_tokens + cache_creation_input_tokens`, which matches the meaning of OpenAI's `prompt_tokens`.
 * `llm_completion_tokens`: Number of chat completion tokens in the response.
-* `llm_total_tokens`: Total number of tokens used (prompt plus completion).
+* `llm_total_tokens`: Total number of tokens used (prompt plus completion, including cache tokens).
 * `llm_cache_read_input_tokens`: Number of input tokens read from cache.
 * `llm_cache_creation_input_tokens`: Number of input tokens written to cache.
 * `llm_reasoning_tokens`: Number of reasoning tokens generated.

docs/en/latest/plugins/ai-rate-limiting.md

@@ -47,7 +47,7 @@ The `ai-rate-limiting` Plugin enforces token-based rate limiting for requests se
 | limit | integer | False | | >0 | The maximum number of tokens allowed within a given time interval. At least one of `limit` and `instances.limit` should be configured. Required if `rules` is not configured. |
 | time_window | integer | False | | >0 | The time interval corresponding to the rate limiting `limit` in seconds. At least one of `time_window` and `instances.time_window` should be configured. Required if `rules` is not configured. |
 | show_limit_quota_header | boolean | False | true | | If true, includes rate limiting response headers. When `rules` is not set, the headers are `X-AI-RateLimit-Limit-*`, `X-AI-RateLimit-Remaining-*`, and `X-AI-RateLimit-Reset-*`, where `*` is the instance name. When `rules` is set, see `rules.header_prefix` for details. |
-| limit_strategy | string | False | total_tokens | [`total_tokens`, `prompt_tokens`, `completion_tokens`, `expression`] | Type of token to apply rate limiting. `total_tokens` is the sum of `prompt_tokens` and `completion_tokens`. When set to `expression`, the `cost_expr` field is used to dynamically calculate token cost. |
+| limit_strategy | string | False | total_tokens | [`total_tokens`, `prompt_tokens`, `completion_tokens`, `expression`] | Type of token to apply rate limiting. `total_tokens` is the sum of `prompt_tokens` and `completion_tokens`. `prompt_tokens` includes tokens read from and written to the prompt cache, for the Anthropic Messages API as well as OpenAI-compatible APIs. When set to `expression`, the `cost_expr` field is used to dynamically calculate token cost. |
 | cost_expr | string | False | | | Lua arithmetic expression for dynamic token cost calculation. Variables are injected from the LLM API raw usage response fields. Nested fields are referenced by joining the parent and child keys with `__`, e.g. `input_tokens_details__cached_tokens` for `input_tokens_details.cached_tokens`. Arrays are skipped. Missing variables default to 0. Only valid when `limit_strategy` is `expression`. Example: `input_tokens + cache_creation_input_tokens + output_tokens`. |
 | instances | array[object] | False | | | LLM instance rate limiting configurations. |
 | instances.name | string | True | | | Name of the LLM service instance. |

docs/en/latest/plugins/prometheus.md

@@ -333,6 +333,8 @@ APISIX increments this gauge when an LLM upstream attempt starts and decrements
 
 ### Labels for `apisix_llm_prompt_tokens`
 
+Prompt tokens include tokens read from and written to the prompt cache. For the Anthropic Messages API, this is `input_tokens + cache_read_input_tokens + cache_creation_input_tokens`, which matches the meaning of OpenAI's `prompt_tokens`.
+
 | Name | Description                                                                                                                   |
 | ---------- | ----------------------------------------------------------------------------------------------------------------------------- |
 | route_id      | ID of the Route that the metric corresponds to when `prefer_name` is `false` (default), and name of the Route when `prefer_name` is `true`. Defaults to an empty string if no Route is matched. |

docs/zh/latest/plugins/ai-proxy-multi.md

@@ -2732,9 +2732,9 @@ curl "http://127.0.0.1:9080/anything" -X POST \
 * `request_type`：请求类型，值可能是 `traditional_http`、`ai_chat` 或 `ai_stream`。
 * `llm_time_to_first_token`：从发送请求到从 LLM 服务接收第一个令牌的持续时间（毫秒）。当 LLM 服务返回错误响应时，该值为收到错误响应为止的持续时间；当完全无法连接到 LLM 服务时，该值保持为 `0`。
 * `llm_model`：LLM 模型。
-* `llm_prompt_tokens`：提示中的令牌数量。
+* `llm_prompt_tokens`：提示中的令牌数量，包含从缓存读取和写入缓存的令牌。对于 Anthropic Messages API，其值为 `input_tokens + cache_read_input_tokens + cache_creation_input_tokens`，与 OpenAI 的 `prompt_tokens` 含义一致。
 * `llm_completion_tokens`：响应中的聊天完成令牌数量。
-* `llm_total_tokens`：使用的总令牌数（提示加完成）。
+* `llm_total_tokens`：使用的总令牌数（提示加完成，包含缓存令牌）。
 * `llm_cache_read_input_tokens`：从缓存读取的输入令牌数量。
 * `llm_cache_creation_input_tokens`：写入缓存的输入令牌数量。
 * `llm_reasoning_tokens`：生成的推理令牌数量。

docs/zh/latest/plugins/ai-proxy.md

@@ -2080,9 +2080,9 @@ curl "http://127.0.0.1:9080/anything" -X POST \
 * `request_type`：请求类型，值可能是 `traditional_http`、`ai_chat` 或 `ai_stream`。
 * `llm_time_to_first_token`：从发送请求到从 LLM 服务接收第一个令牌的持续时间（毫秒）。当 LLM 服务返回错误响应时，该值为收到错误响应为止的持续时间；当完全无法连接到 LLM 服务时，该值保持为 `0`。
 * `llm_model`：LLM 模型。
-* `llm_prompt_tokens`：提示中的令牌数量。
+* `llm_prompt_tokens`：提示中的令牌数量，包含从缓存读取和写入缓存的令牌。对于 Anthropic Messages API，其值为 `input_tokens + cache_read_input_tokens + cache_creation_input_tokens`，与 OpenAI 的 `prompt_tokens` 含义一致。
 * `llm_completion_tokens`：响应中的聊天完成令牌数量。
-* `llm_total_tokens`：使用的总令牌数（提示加完成）。
+* `llm_total_tokens`：使用的总令牌数（提示加完成，包含缓存令牌）。
 * `llm_cache_read_input_tokens`：从缓存读取的输入令牌数量。
 * `llm_cache_creation_input_tokens`：写入缓存的输入令牌数量。
 * `llm_reasoning_tokens`：生成的推理令牌数量。