From 924038b4b1bab0f893dc4ecb1b3bee6eb3ba4033 Mon Sep 17 00:00:00 2001 From: Sergei Bronnikov Date: Thu, 8 Oct 2026 17:56:18 +0100 Subject: [PATCH 1/4] update models --- internal/message/handler.go | 2 +- internal/provider/anthropic/cost.go | 117 +++++++++++++++++++--------- internal/provider/openai/cost.go | 76 +++++++++++------- internal/provider/utils.go | 42 ++++++++++ 4 files changed, 175 insertions(+), 62 deletions(-) create mode 100644 internal/provider/utils.go diff --git a/internal/message/handler.go b/internal/message/handler.go index 3a82bee..df2afba 100644 --- a/internal/message/handler.go +++ b/internal/message/handler.go @@ -703,7 +703,7 @@ func (h *Handler) decorateEvent(m Message) error { e.Event.CostInUsd = cost + completionCost if e.CostMap != nil { - model := openai.ModelWithContextLength(e.Event.Model, int64(tks+completiontks)) + model := provider.ModelWithContextLength(e.Event.Model, int64(tks+completiontks)) newCost, err := provider.EstimateTotalCostWithCostMaps(model, tks, completiontks, 1000, e.CostMap.PromptCostPerModel, e.CostMap.CompletionCostPerModel) if err != nil { h.log.Debug("error when estimating total cost with cost maps", zap.Error(err)) diff --git a/internal/provider/anthropic/cost.go b/internal/provider/anthropic/cost.go index 896b92c..2fad5ee 100644 --- a/internal/provider/anthropic/cost.go +++ b/internal/provider/anthropic/cost.go @@ -1,13 +1,22 @@ package anthropic import ( - "errors" "fmt" "strings" + + "github.com/bricks-cloud/bricksllm/internal/provider" ) var AnthropicPerMillionTokenCost = map[string]map[string]float64{ "prompt": { + "claude-mythos-5-1": 10.0, + "claude-mythos-5": 10.0, + + "claude-fable-5-1": 10.0, + "claude-fable-5": 10.0, + + "claude-opus-5-5": 4.0, + "claude-opus-5": 5.0, "claude-opus-4-8": 5.0, "claude-opus-4-7": 5.0, "claude-opus-4-6": 5.0, @@ -15,11 +24,16 @@ var AnthropicPerMillionTokenCost = map[string]map[string]float64{ "claude-opus-4-1": 15.0, "claude-opus-4": 15.0, + "claude-sonnet-5-5": 2.0, + "claude-sonnet-5": 2.0, "claude-sonnet-4-6": 3.0, "claude-sonnet-4-5": 3.0, "claude-sonnet-4": 3.0, - "claude-haiku-4-5": 1.0, + "claude-haiku-5-5": 0.5, + "claude-haiku-5-5~long": 0.5, + "claude-haiku-5-5~short": 0.1, + "claude-haiku-4-5": 1.0, // old models "claude-3-7-sonnet": 3.0, @@ -29,6 +43,14 @@ var AnthropicPerMillionTokenCost = map[string]map[string]float64{ "claude-3-opus": 15.0, }, "completion": { + "claude-mythos-5-1": 50.0, + "claude-mythos-5": 50.0, + + "claude-fable-5-1": 50.0, + "claude-fable-5": 50.0, + + "claude-opus-5-5": 20.0, + "claude-opus-5": 25.0, "claude-opus-4-8": 25.0, "claude-opus-4-7": 25.0, "claude-opus-4-6": 25.0, @@ -36,11 +58,16 @@ var AnthropicPerMillionTokenCost = map[string]map[string]float64{ "claude-opus-4-1": 75.0, "claude-opus-4": 75.0, + "claude-sonnet-5-5": 10.0, + "claude-sonnet-5": 10.0, "claude-sonnet-4-6": 15.0, "claude-sonnet-4-5": 15.0, "claude-sonnet-4": 15.0, - "claude-haiku-4-5": 5.0, + "claude-haiku-5-5": 2.5, + "claude-haiku-5-5~long": 2.5, + "claude-haiku-5-5~short": 0.5, + "claude-haiku-4-5": 5.0, // old models "claude-3-7-sonnet": 15.0, @@ -68,12 +95,15 @@ func NewCostEstimator(tc tokenCounter) *CostEstimator { } func (ce *CostEstimator) EstimateTotalCost(model string, promptTks, completionTks int) (float64, error) { - promptCost, err := ce.EstimatePromptCost(model, promptTks) + normalized := normalizeAnthropicModel(model) + modelWithCtx := provider.ModelWithContextLength(normalized, int64(promptTks+completionTks)) + + promptCost, err := ce.estimateCost("prompt", modelWithCtx, promptTks) if err != nil { return 0, err } - completionCost, err := ce.EstimateCompletionCost(model, completionTks) + completionCost, err := ce.estimateCost("completion", modelWithCtx, completionTks) if err != nil { return 0, err } @@ -81,21 +111,23 @@ func (ce *CostEstimator) EstimateTotalCost(model string, promptTks, completionTk return promptCost + completionCost, nil } -func (ce *CostEstimator) EstimatePromptCost(model string, tks int) (float64, error) { - costMap, ok := ce.tokenCostMap["prompt"] - if !ok { - return 0, errors.New("prompt token cost is not provided") - +// normalizeAnthropicModel strips version/date suffixes (and Bedrock's +// cross-region "us.anthropic...." prefix) down to the base model family name +// used as a key in AnthropicPerMillionTokenCost. +func normalizeAnthropicModel(model string) string { + if strings.HasPrefix(model, "us") { + return convertAmazonModelToAnthropicModel(model) } + return SelectModel(model) +} - selected := "" - if strings.HasPrefix(model, "us") { - selected = convertAmazonModelToAnthropicModel(model) - } else { - selected = SelectModel(model) +func (ce *CostEstimator) estimateCost(costMapKey, model string, tks int) (float64, error) { + costMap, ok := ce.tokenCostMap[costMapKey] + if !ok { + return 0, fmt.Errorf("%s token cost is not provided", costMapKey) } - cost, ok := costMap[selected] + cost, ok := costMap[model] if !ok { return 0, fmt.Errorf("%s is not present in the cost map provided", model) } @@ -104,7 +136,30 @@ func (ce *CostEstimator) EstimatePromptCost(model string, tks int) (float64, err return tksInFloat / 1000000 * cost, nil } +func (ce *CostEstimator) EstimatePromptCost(model string, tks int) (float64, error) { + modelWithCtx := provider.ModelWithContextLength(normalizeAnthropicModel(model), int64(tks)) + return ce.estimateCost("prompt", modelWithCtx, tks) +} + func SelectModel(model string) string { + if strings.HasPrefix(model, "claude-mythos-5-1") { + return "claude-mythos-5-1" + } + if strings.HasPrefix(model, "claude-mythos-5") { + return "claude-mythos-5" + } + if strings.HasPrefix(model, "claude-fable-5-1") { + return "claude-fable-5-1" + } + if strings.HasPrefix(model, "claude-fable-5") { + return "claude-fable-5" + } + if strings.HasPrefix(model, "claude-opus-5-5") { + return "claude-opus-5-5" + } + if strings.HasPrefix(model, "claude-opus-5") { + return "claude-opus-5" + } if strings.HasPrefix(model, "claude-opus-4-8") { return "claude-opus-4-8" } @@ -123,6 +178,12 @@ func SelectModel(model string) string { if strings.HasPrefix(model, "claude-opus-4") { return "claude-opus-4" } + if strings.HasPrefix(model, "claude-sonnet-5-5") { + return "claude-sonnet-5-5" + } + if strings.HasPrefix(model, "claude-sonnet-5") { + return "claude-sonnet-5" + } if strings.HasPrefix(model, "claude-sonnet-4-6") { return "claude-sonnet-4-6" } @@ -132,6 +193,9 @@ func SelectModel(model string) string { if strings.HasPrefix(model, "claude-sonnet-4") { return "claude-sonnet-4" } + if strings.HasPrefix(model, "claude-haiku-5-5") { + return "claude-haiku-5-5" + } if strings.HasPrefix(model, "claude-haiku-4-5") { return "claude-haiku-4-5" } @@ -165,25 +229,8 @@ func convertAmazonModelToAnthropicModel(model string) string { } func (ce *CostEstimator) EstimateCompletionCost(model string, tks int) (float64, error) { - costMap, ok := ce.tokenCostMap["completion"] - if !ok { - return 0, errors.New("prompt token cost is not provided") - } - - selected := "" - if strings.HasPrefix(model, "us") { - selected = convertAmazonModelToAnthropicModel(model) - } else { - selected = SelectModel(model) - } - - cost, ok := costMap[selected] - if !ok { - return 0, errors.New("model is not present in the cost map provided") - } - - tksInFloat := float64(tks) - return tksInFloat / 1000000 * cost, nil + modelWithCtx := provider.ModelWithContextLength(normalizeAnthropicModel(model), int64(tks)) + return ce.estimateCost("completion", modelWithCtx, tks) } func (ce *CostEstimator) Count(input string) int { diff --git a/internal/provider/openai/cost.go b/internal/provider/openai/cost.go index 93fd7ff..d11fd71 100644 --- a/internal/provider/openai/cost.go +++ b/internal/provider/openai/cost.go @@ -8,6 +8,7 @@ import ( "slices" "strings" + "github.com/bricks-cloud/bricksllm/internal/provider" "github.com/bricks-cloud/bricksllm/internal/util" responsesOpenai "github.com/openai/openai-go/responses" goopenai "github.com/sashabaranov/go-openai" @@ -41,6 +42,21 @@ var OpenAiPerThousandTokenCost = map[string]map[string]float64{ "chatgpt-image-latest": 0.005, "gpt-image-1-mini": 0.002, + "gpt-6-astra": 0.020, + "gpt-6-astra~long": 0.020, + "gpt-6-astra~short": 0.010, + "gpt-6.1-sol": 0.004, + "gpt-6.1-sol~long": 0.004, + "gpt-6.1-sol~short": 0.002, + "gpt-6-luna": 0.0002, + "gpt-6-luna~long": 0.0002, + "gpt-6-luna~short": 0.0001, + + "gpt-5.6-sol": 0.008, + "gpt-5.6-sol~long": 0.008, + "gpt-5.6-sol~short": 0.004, + "gpt-5.6-cyber": 0.0125, + "gpt-5.5": 0.01, "gpt-5.5~long": 0.01, "gpt-5.5~short": 0.005, @@ -122,6 +138,21 @@ var OpenAiPerThousandTokenCost = map[string]map[string]float64{ "babbage-002": 0.000400, }, "cached-prompt": { + "gpt-6-astra": 0.002, + "gpt-6-astra~long": 0.002, + "gpt-6-astra~short": 0.001, + "gpt-6.1-sol": 0.0002, + "gpt-6.1-sol~long": 0.0002, + "gpt-6.1-sol~short": 0.0001, + "gpt-6-luna": 0.00002, + "gpt-6-luna~long": 0.00002, + "gpt-6-luna~short": 0.00001, + + "gpt-5.6-sol": 0.0008, + "gpt-5.6-sol~long": 0.0008, + "gpt-5.6-sol~short": 0.0004, + "gpt-5.6-cyber": 0.00125, + "gpt-5.5": 0.001, "gpt-5.5~long": 0.001, "gpt-5.5~short": 0.0005, @@ -213,6 +244,21 @@ var OpenAiPerThousandTokenCost = map[string]map[string]float64{ "gpt-image-1.5": 0.010, "chatgpt-image-latest": 0.010, + "gpt-6-astra": 0.075, + "gpt-6-astra~long": 0.075, + "gpt-6-astra~short": 0.05, + "gpt-6.1-sol": 0.015, + "gpt-6.1-sol~long": 0.015, + "gpt-6.1-sol~short": 0.01, + "gpt-6-luna": 0.00075, + "gpt-6-luna~long": 0.00075, + "gpt-6-luna~short": 0.0005, + + "gpt-5.6-sol": 0.03, + "gpt-5.6-sol~long": 0.03, + "gpt-5.6-sol~short": 0.02, + "gpt-5.6-cyber": 0.075, + "gpt-5.5": 0.045, "gpt-5.5~long": 0.045, "gpt-5.5~short": 0.03, @@ -425,7 +471,7 @@ func NewCostEstimator(m map[string]map[string]float64, tc tokenCounter) *CostEst func (ce *CostEstimator) EstimateTotalCost(model string, promptTks, completionTks int) (float64, error) { totalTokens := int64(promptTks + completionTks) - model = ModelWithContextLength(model, totalTokens) + model = provider.ModelWithContextLength(model, totalTokens) promptCost, err := ce.EstimatePromptCost(model, promptTks) if err != nil { return 0, err @@ -505,7 +551,7 @@ func (ce *CostEstimator) EstimateChatCompletionPromptCostWithTokenCounts(r *goop return 0, 0, err } - model := ModelWithContextLength(r.Model, int64(tks)) + model := provider.ModelWithContextLength(r.Model, int64(tks)) cost, err := ce.EstimatePromptCost(model, tks) if err != nil { return 0, 0, err @@ -524,7 +570,7 @@ func (ce *CostEstimator) EstimateChatCompletionStreamCostWithTokenCounts(model s return 0, 0, err } - model = ModelWithContextLength(model, int64(tks)) + model = provider.ModelWithContextLength(model, int64(tks)) cost, err := ce.EstimateCompletionCost(model, tks) if err != nil { return 0, 0, err @@ -850,7 +896,7 @@ func (ce *CostEstimator) EstimateResponseApiTotalCost(model string, usage respon outputTokens := usage.OutputTokens totalTokens := inputTokens + cachedInputTokens + outputTokens - model = ModelWithContextLength(model, totalTokens) + model = provider.ModelWithContextLength(model, totalTokens) cachedInputCost, err := ce.estimateResponseApiTokensCost("cached-prompt", model, cachedInputTokens) if err != nil { @@ -1110,25 +1156,3 @@ func countTotalTokens(model string, r *goopenai.ChatCompletionRequest, tc tokenC return tks + ftks + mtks, err } - -var modelWithLengthCtx = []string{ - "gpt-5.5", - "gpt-5.5-pro", - "gpt-5.4", - "gpt-5.4-pro", -} - -func ModelWithContextLength(model string, tokens int64) string { - trimmed := strings.TrimSpace(model) - if slices.Contains(modelWithLengthCtx, trimmed) { - return trimmed + contextLengthSuffixByTokens(tokens) - } - return trimmed -} - -func contextLengthSuffixByTokens(tokens int64) string { - if tokens >= 272000 { - return "~long" - } - return "~short" -} diff --git a/internal/provider/utils.go b/internal/provider/utils.go new file mode 100644 index 0000000..3203341 --- /dev/null +++ b/internal/provider/utils.go @@ -0,0 +1,42 @@ +package provider + +import ( + "strings" +) + +type ModelContextLenght struct { + Model string + Tokens int64 +} + +var modelWithLengthCtx = []ModelContextLenght{ + // open ai + ModelContextLenght{Model: "gpt-6-astra", Tokens: 272000}, + ModelContextLenght{Model: "gpt-6.1-sol", Tokens: 272000}, + ModelContextLenght{Model: "gpt-6-luna", Tokens: 272000}, + ModelContextLenght{Model: "gpt-5.6-sol", Tokens: 272000}, + ModelContextLenght{Model: "gpt-5.5", Tokens: 272000}, + ModelContextLenght{Model: "gpt-5.5-pro", Tokens: 272000}, + ModelContextLenght{Model: "gpt-5.4", Tokens: 272000}, + ModelContextLenght{Model: "gpt-5.4-pro", Tokens: 272000}, + + // anthropic + ModelContextLenght{Model: "claude-haiku-5-5", Tokens: 100000}, +} + +func ModelWithContextLength(model string, tokens int64) string { + trueModel := strings.TrimSpace(model) + for _, m := range modelWithLengthCtx { + if trueModel == m.Model { + return trueModel + contextLengthSuffixByTokens(tokens, m.Tokens) + } + } + return trueModel +} + +func contextLengthSuffixByTokens(tokens, threshold int64) string { + if tokens >= threshold { + return "~long" + } + return "~short" +} From 0e6db266568ef00aff44cbc7399c28da220dda83 Mon Sep 17 00:00:00 2001 From: Sergei Bronnikov Date: Fri, 9 Oct 2026 10:40:38 +0100 Subject: [PATCH 2/4] review --- internal/message/handler.go | 6 +++--- internal/provider/anthropic/cost.go | 8 ++++---- internal/provider/utils.go | 2 +- internal/server/web/proxy/anthropic.go | 4 ++-- internal/server/web/proxy/bedrock.go | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/internal/message/handler.go b/internal/message/handler.go index df2afba..aeef31e 100644 --- a/internal/message/handler.go +++ b/internal/message/handler.go @@ -24,7 +24,7 @@ import ( type anthropicEstimator interface { EstimateTotalCost(model string, promptTks, completionTks int) (float64, error) - EstimateCompletionCost(model string, tks int) (float64, error) + EstimateCompletionCost(model string, promptTks, completionTks int) (float64, error) EstimatePromptCost(model string, tks int) (float64, error) Count(input string) int } @@ -523,7 +523,7 @@ func (h *Handler) decorateEvent(m Message) error { completiontks := h.ae.Count(e.Content) completiontks += anthropicCompletionMagicNum - completionCost, err := h.ae.EstimateCompletionCost(model, completiontks) + completionCost, err := h.ae.EstimateCompletionCost(model, tks, completiontks) if err != nil { telemetry.Incr("bricksllm.message.handler.decorate_event.estimate_completion_cost_error", nil, 1) return err @@ -562,7 +562,7 @@ func (h *Handler) decorateEvent(m Message) error { completiontks := h.ae.Count(e.Content) completiontks += anthropicCompletionMagicNum - completionCost, err := h.ae.EstimateCompletionCost(translatedModel, completiontks) + completionCost, err := h.ae.EstimateCompletionCost(translatedModel, tks, completiontks) if err != nil { telemetry.Incr("bricksllm.message.handler.decorate_event.estimate_completion_cost_error", nil, 1) return err diff --git a/internal/provider/anthropic/cost.go b/internal/provider/anthropic/cost.go index 2fad5ee..bbdff3a 100644 --- a/internal/provider/anthropic/cost.go +++ b/internal/provider/anthropic/cost.go @@ -96,7 +96,7 @@ func NewCostEstimator(tc tokenCounter) *CostEstimator { func (ce *CostEstimator) EstimateTotalCost(model string, promptTks, completionTks int) (float64, error) { normalized := normalizeAnthropicModel(model) - modelWithCtx := provider.ModelWithContextLength(normalized, int64(promptTks+completionTks)) + modelWithCtx := provider.ModelWithContextLength(normalized, int64(promptTks)) promptCost, err := ce.estimateCost("prompt", modelWithCtx, promptTks) if err != nil { @@ -228,9 +228,9 @@ func convertAmazonModelToAnthropicModel(model string) string { return SelectModel(parts[2]) } -func (ce *CostEstimator) EstimateCompletionCost(model string, tks int) (float64, error) { - modelWithCtx := provider.ModelWithContextLength(normalizeAnthropicModel(model), int64(tks)) - return ce.estimateCost("completion", modelWithCtx, tks) +func (ce *CostEstimator) EstimateCompletionCost(model string, promptTks, completionTks int) (float64, error) { + modelWithCtx := provider.ModelWithContextLength(normalizeAnthropicModel(model), int64(promptTks)) + return ce.estimateCost("completion", modelWithCtx, completionTks) } func (ce *CostEstimator) Count(input string) int { diff --git a/internal/provider/utils.go b/internal/provider/utils.go index 3203341..f4aedc6 100644 --- a/internal/provider/utils.go +++ b/internal/provider/utils.go @@ -21,7 +21,7 @@ var modelWithLengthCtx = []ModelContextLenght{ ModelContextLenght{Model: "gpt-5.4-pro", Tokens: 272000}, // anthropic - ModelContextLenght{Model: "claude-haiku-5-5", Tokens: 100000}, + ModelContextLenght{Model: "claude-haiku-5-5", Tokens: 100001}, } func ModelWithContextLength(model string, tokens int64) string { diff --git a/internal/server/web/proxy/anthropic.go b/internal/server/web/proxy/anthropic.go index 79b479c..8c70d9e 100644 --- a/internal/server/web/proxy/anthropic.go +++ b/internal/server/web/proxy/anthropic.go @@ -22,7 +22,7 @@ import ( type anthropicEstimator interface { EstimateTotalCost(model string, promptTks, completionTks int) (float64, error) - EstimateCompletionCost(model string, tks int) (float64, error) + EstimateCompletionCost(model string, promptTks, completionTks int) (float64, error) EstimatePromptCost(model string, tks int) (float64, error) Count(input string) int CountMessagesTokens(messages []anthropic.Message) int @@ -425,7 +425,7 @@ func getMessagesHandler(prod, private bool, client http.Client, e anthropicEstim defer func() { tks := response.Usage.OutputTokens model := c.GetString("model") - cost, err := e.EstimateCompletionCost(model, tks) + cost, err := e.EstimateCompletionCost(model, response.Usage.InputTokens, tks) if err != nil { telemetry.Incr("bricksllm.proxy.get_messages_handler.estimate_messages_cost_error", nil, 1) logError(log, "error when estimating anthropic messages stream cost", prod, err) diff --git a/internal/server/web/proxy/bedrock.go b/internal/server/web/proxy/bedrock.go index d9dc977..b1b6f2e 100644 --- a/internal/server/web/proxy/bedrock.go +++ b/internal/server/web/proxy/bedrock.go @@ -162,7 +162,7 @@ func getBedrockCompletionHandler(prod bool, e anthropicEstimator) gin.HandlerFun defer func() { model := c.GetString("model") translatedModel := util.TranslateBedrockModelToAnthropicModel(model) - compeltionCost, err := e.EstimateCompletionCost(translatedModel, completionTokenCount) + compeltionCost, err := e.EstimateCompletionCost(translatedModel, promptTokenCount, completionTokenCount) if err != nil { telemetry.Incr("bricksllm.proxy.get_bedrock_completion_handler.estimate_completion_cost_error", nil, 1) logError(log, "error when estimating bedrock completion cost", prod, err) @@ -437,7 +437,7 @@ func getBedrockMessagesHandler(prod bool, e anthropicEstimator) gin.HandlerFunc defer func() { model := c.GetString("model") translatedModel := util.TranslateBedrockModelToAnthropicModel(model) - compeltionCost, err := e.EstimateCompletionCost(translatedModel, completionTokenCount) + compeltionCost, err := e.EstimateCompletionCost(translatedModel, promptTokenCount, completionTokenCount) if err != nil { telemetry.Incr("bricksllm.proxy.get_bedrock_messages_handler.estimate_completion_cost_error", nil, 1) logError(log, "error when estimating bedrock completion cost", prod, err) From eecb411c85cf13eed471a0a901aa9e8605732920 Mon Sep 17 00:00:00 2001 From: Sergei Bronnikov Date: Fri, 9 Oct 2026 16:36:34 +0100 Subject: [PATCH 3/4] review --- internal/message/handler.go | 9 +++--- internal/provider/openai/cost.go | 40 ++++++++++--------------- internal/server/web/proxy/middleware.go | 3 +- 3 files changed, 21 insertions(+), 31 deletions(-) diff --git a/internal/message/handler.go b/internal/message/handler.go index aeef31e..cf75a48 100644 --- a/internal/message/handler.go +++ b/internal/message/handler.go @@ -33,9 +33,8 @@ type estimator interface { EstimateCompletionsRequestCostWithTokenCounts(model string, content any) (int, float64, error) EstimateCompletionsStreamCostWithTokenCounts(model string, content string) (int, float64, error) EstimateSpeechCost(input string, model string) (float64, error) - EstimateChatCompletionPromptCostWithTokenCounts(r *goopenai.ChatCompletionRequest) (int, float64, error) EstimateEmbeddingsCost(r *goopenai.EmbeddingRequest) (float64, error) - EstimateChatCompletionStreamCostWithTokenCounts(model, content string) (int, float64, error) + EstimateChatCompletionStreamCostWithTokenCounts(model string, promptTks int, content string) (int, float64, float64, error) EstimateCompletionCost(model string, tks int) (float64, error) EstimateTotalCost(model string, promptTks, completionTks int) (float64, error) EstimateEmbeddingsInputCost(model string, tks int) (float64, error) @@ -684,13 +683,13 @@ func (h *Handler) decorateEvent(m Message) error { } if ccr.Stream { - tks, cost, err := h.e.EstimateChatCompletionPromptCostWithTokenCounts(ccr) + tks, err := h.e.EstimateChatCompletionPromptTokenCounts(ccr.Model, ccr) if err != nil { - telemetry.Incr("bricksllm.message.handler.decorate_event.estimate_chat_completion_prompt_cost_with_token_counts", nil, 1) + telemetry.Incr("bricksllm.message.handler.decorate_event.estimate_chat_completion_prompt_token_counts", nil, 1) return err } - completiontks, completionCost, err := h.e.EstimateChatCompletionStreamCostWithTokenCounts(e.Event.Model, e.Content) + completiontks, cost, completionCost, err := h.e.EstimateChatCompletionStreamCostWithTokenCounts(e.Event.Model, tks, e.Content) if err != nil { telemetry.Incr("bricksllm.message.handler.decorate_event.estimate_chat_completion_stream_cost_with_token_counts", nil, 1) return err diff --git a/internal/provider/openai/cost.go b/internal/provider/openai/cost.go index d11fd71..0b49542 100644 --- a/internal/provider/openai/cost.go +++ b/internal/provider/openai/cost.go @@ -541,42 +541,34 @@ func (ce *CostEstimator) EstimateChatCompletionPromptTokenCounts(model string, r return tks, nil } -func (ce *CostEstimator) EstimateChatCompletionPromptCostWithTokenCounts(r *goopenai.ChatCompletionRequest) (int, float64, error) { - if len(r.Model) == 0 { - return 0, 0, errors.New("model is not provided") - } - - tks, err := countTotalTokens(r.Model, r, ce.tc) - if err != nil { - return 0, 0, err +// EstimateChatCompletionStreamCostWithTokenCounts estimates the cost of a +// streamed chat completion. promptTks must be the already-counted prompt +// token count so that the long/short context tier is picked using +// promptTks+completionTks, matching the tiering used by EstimateTotalCost +// for non-streaming requests. +func (ce *CostEstimator) EstimateChatCompletionStreamCostWithTokenCounts(model string, promptTks int, content string) (int, float64, float64, error) { + if len(model) == 0 { + return 0, 0, 0, errors.New("model is not provided") } - model := provider.ModelWithContextLength(r.Model, int64(tks)) - cost, err := ce.EstimatePromptCost(model, tks) + tks, err := ce.tc.Count(model, content) if err != nil { - return 0, 0, err + return 0, 0, 0, err } - return tks, cost, nil -} - -func (ce *CostEstimator) EstimateChatCompletionStreamCostWithTokenCounts(model string, content string) (int, float64, error) { - if len(model) == 0 { - return 0, 0, errors.New("model is not provided") - } + tieredModel := provider.ModelWithContextLength(model, int64(promptTks+tks)) - tks, err := ce.tc.Count(model, content) + promptCost, err := ce.EstimatePromptCost(tieredModel, promptTks) if err != nil { - return 0, 0, err + return 0, 0, 0, err } - model = provider.ModelWithContextLength(model, int64(tks)) - cost, err := ce.EstimateCompletionCost(model, tks) + completionCost, err := ce.EstimateCompletionCost(tieredModel, tks) if err != nil { - return 0, 0, err + return 0, 0, 0, err } - return tks, cost, nil + return tks, promptCost, completionCost, nil } func (ce *CostEstimator) EstimateCompletionsRequestCostWithTokenCounts(model string, content any) (int, float64, error) { diff --git a/internal/server/web/proxy/middleware.go b/internal/server/web/proxy/middleware.go index a18565a..674497a 100644 --- a/internal/server/web/proxy/middleware.go +++ b/internal/server/web/proxy/middleware.go @@ -52,9 +52,8 @@ type keyStorage interface { type estimator interface { EstimateTranscriptionCost(secs float64, model string, usage *openai.TranscriptionResponseUsage) (float64, error) EstimateSpeechCost(input string, model string) (float64, error) - EstimateChatCompletionPromptCostWithTokenCounts(r *goopenai.ChatCompletionRequest) (int, float64, error) EstimateEmbeddingsCost(r *goopenai.EmbeddingRequest) (float64, error) - EstimateChatCompletionStreamCostWithTokenCounts(model, content string) (int, float64, error) + EstimateChatCompletionStreamCostWithTokenCounts(model string, promptTks int, content string) (int, float64, float64, error) EstimateCompletionCost(model string, tks int) (float64, error) EstimateTotalCost(model string, promptTks, completionTks int) (float64, error) EstimateEmbeddingsInputCost(model string, tks int) (float64, error) From d813213462ed27abf1ab0200f0fa39e446f97e4a Mon Sep 17 00:00:00 2001 From: Sergei Bronnikov Date: Fri, 9 Oct 2026 17:11:38 +0100 Subject: [PATCH 4/4] review --- internal/message/handler.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/message/handler.go b/internal/message/handler.go index cf75a48..b645f99 100644 --- a/internal/message/handler.go +++ b/internal/message/handler.go @@ -704,11 +704,13 @@ func (h *Handler) decorateEvent(m Message) error { if e.CostMap != nil { model := provider.ModelWithContextLength(e.Event.Model, int64(tks+completiontks)) newCost, err := provider.EstimateTotalCostWithCostMaps(model, tks, completiontks, 1000, e.CostMap.PromptCostPerModel, e.CostMap.CompletionCostPerModel) + if err != nil { + newCost, err = provider.EstimateTotalCostWithCostMaps(e.Event.Model, tks, completiontks, 1000, e.CostMap.PromptCostPerModel, e.CostMap.CompletionCostPerModel) + } if err != nil { h.log.Debug("error when estimating total cost with cost maps", zap.Error(err)) telemetry.Incr("bricksllm.proxy.decorate_event.estimate_total_cost_with_cost_maps_error", nil, 1) - } - + } if newCost != 0 { e.Event.CostInUsd = newCost }