diff --git a/internal/message/handler.go b/internal/message/handler.go index 3a82bee..b645f99 100644 --- a/internal/message/handler.go +++ b/internal/message/handler.go @@ -24,7 +24,7 @@ import ( type anthropicEstimator interface { EstimateTotalCost(model string, promptTks, completionTks int) (float64, error) - EstimateCompletionCost(model string, tks int) (float64, error) + EstimateCompletionCost(model string, promptTks, completionTks int) (float64, error) EstimatePromptCost(model string, tks int) (float64, error) Count(input string) int } @@ -33,9 +33,8 @@ type estimator interface { EstimateCompletionsRequestCostWithTokenCounts(model string, content any) (int, float64, error) EstimateCompletionsStreamCostWithTokenCounts(model string, content string) (int, float64, error) EstimateSpeechCost(input string, model string) (float64, error) - EstimateChatCompletionPromptCostWithTokenCounts(r *goopenai.ChatCompletionRequest) (int, float64, error) EstimateEmbeddingsCost(r *goopenai.EmbeddingRequest) (float64, error) - EstimateChatCompletionStreamCostWithTokenCounts(model, content string) (int, float64, error) + EstimateChatCompletionStreamCostWithTokenCounts(model string, promptTks int, content string) (int, float64, float64, error) EstimateCompletionCost(model string, tks int) (float64, error) EstimateTotalCost(model string, promptTks, completionTks int) (float64, error) EstimateEmbeddingsInputCost(model string, tks int) (float64, error) @@ -523,7 +522,7 @@ func (h *Handler) decorateEvent(m Message) error { completiontks := h.ae.Count(e.Content) completiontks += anthropicCompletionMagicNum - completionCost, err := h.ae.EstimateCompletionCost(model, completiontks) + completionCost, err := h.ae.EstimateCompletionCost(model, tks, completiontks) if err != nil { telemetry.Incr("bricksllm.message.handler.decorate_event.estimate_completion_cost_error", nil, 1) return err @@ -562,7 +561,7 @@ func (h *Handler) decorateEvent(m Message) error { completiontks := h.ae.Count(e.Content) completiontks += anthropicCompletionMagicNum - completionCost, err := h.ae.EstimateCompletionCost(translatedModel, completiontks) + completionCost, err := h.ae.EstimateCompletionCost(translatedModel, tks, completiontks) if err != nil { telemetry.Incr("bricksllm.message.handler.decorate_event.estimate_completion_cost_error", nil, 1) return err @@ -684,13 +683,13 @@ func (h *Handler) decorateEvent(m Message) error { } if ccr.Stream { - tks, cost, err := h.e.EstimateChatCompletionPromptCostWithTokenCounts(ccr) + tks, err := h.e.EstimateChatCompletionPromptTokenCounts(ccr.Model, ccr) if err != nil { - telemetry.Incr("bricksllm.message.handler.decorate_event.estimate_chat_completion_prompt_cost_with_token_counts", nil, 1) + telemetry.Incr("bricksllm.message.handler.decorate_event.estimate_chat_completion_prompt_token_counts", nil, 1) return err } - completiontks, completionCost, err := h.e.EstimateChatCompletionStreamCostWithTokenCounts(e.Event.Model, e.Content) + completiontks, cost, completionCost, err := h.e.EstimateChatCompletionStreamCostWithTokenCounts(e.Event.Model, tks, e.Content) if err != nil { telemetry.Incr("bricksllm.message.handler.decorate_event.estimate_chat_completion_stream_cost_with_token_counts", nil, 1) return err @@ -703,13 +702,15 @@ func (h *Handler) decorateEvent(m Message) error { e.Event.CostInUsd = cost + completionCost if e.CostMap != nil { - model := openai.ModelWithContextLength(e.Event.Model, int64(tks+completiontks)) + model := provider.ModelWithContextLength(e.Event.Model, int64(tks+completiontks)) newCost, err := provider.EstimateTotalCostWithCostMaps(model, tks, completiontks, 1000, e.CostMap.PromptCostPerModel, e.CostMap.CompletionCostPerModel) + if err != nil { + newCost, err = provider.EstimateTotalCostWithCostMaps(e.Event.Model, tks, completiontks, 1000, e.CostMap.PromptCostPerModel, e.CostMap.CompletionCostPerModel) + } if err != nil { h.log.Debug("error when estimating total cost with cost maps", zap.Error(err)) telemetry.Incr("bricksllm.proxy.decorate_event.estimate_total_cost_with_cost_maps_error", nil, 1) - } - + } if newCost != 0 { e.Event.CostInUsd = newCost } diff --git a/internal/provider/anthropic/cost.go b/internal/provider/anthropic/cost.go index 896b92c..bbdff3a 100644 --- a/internal/provider/anthropic/cost.go +++ b/internal/provider/anthropic/cost.go @@ -1,13 +1,22 @@ package anthropic import ( - "errors" "fmt" "strings" + + "github.com/bricks-cloud/bricksllm/internal/provider" ) var AnthropicPerMillionTokenCost = map[string]map[string]float64{ "prompt": { + "claude-mythos-5-1": 10.0, + "claude-mythos-5": 10.0, + + "claude-fable-5-1": 10.0, + "claude-fable-5": 10.0, + + "claude-opus-5-5": 4.0, + "claude-opus-5": 5.0, "claude-opus-4-8": 5.0, "claude-opus-4-7": 5.0, "claude-opus-4-6": 5.0, @@ -15,11 +24,16 @@ var AnthropicPerMillionTokenCost = map[string]map[string]float64{ "claude-opus-4-1": 15.0, "claude-opus-4": 15.0, + "claude-sonnet-5-5": 2.0, + "claude-sonnet-5": 2.0, "claude-sonnet-4-6": 3.0, "claude-sonnet-4-5": 3.0, "claude-sonnet-4": 3.0, - "claude-haiku-4-5": 1.0, + "claude-haiku-5-5": 0.5, + "claude-haiku-5-5~long": 0.5, + "claude-haiku-5-5~short": 0.1, + "claude-haiku-4-5": 1.0, // old models "claude-3-7-sonnet": 3.0, @@ -29,6 +43,14 @@ var AnthropicPerMillionTokenCost = map[string]map[string]float64{ "claude-3-opus": 15.0, }, "completion": { + "claude-mythos-5-1": 50.0, + "claude-mythos-5": 50.0, + + "claude-fable-5-1": 50.0, + "claude-fable-5": 50.0, + + "claude-opus-5-5": 20.0, + "claude-opus-5": 25.0, "claude-opus-4-8": 25.0, "claude-opus-4-7": 25.0, "claude-opus-4-6": 25.0, @@ -36,11 +58,16 @@ var AnthropicPerMillionTokenCost = map[string]map[string]float64{ "claude-opus-4-1": 75.0, "claude-opus-4": 75.0, + "claude-sonnet-5-5": 10.0, + "claude-sonnet-5": 10.0, "claude-sonnet-4-6": 15.0, "claude-sonnet-4-5": 15.0, "claude-sonnet-4": 15.0, - "claude-haiku-4-5": 5.0, + "claude-haiku-5-5": 2.5, + "claude-haiku-5-5~long": 2.5, + "claude-haiku-5-5~short": 0.5, + "claude-haiku-4-5": 5.0, // old models "claude-3-7-sonnet": 15.0, @@ -68,12 +95,15 @@ func NewCostEstimator(tc tokenCounter) *CostEstimator { } func (ce *CostEstimator) EstimateTotalCost(model string, promptTks, completionTks int) (float64, error) { - promptCost, err := ce.EstimatePromptCost(model, promptTks) + normalized := normalizeAnthropicModel(model) + modelWithCtx := provider.ModelWithContextLength(normalized, int64(promptTks)) + + promptCost, err := ce.estimateCost("prompt", modelWithCtx, promptTks) if err != nil { return 0, err } - completionCost, err := ce.EstimateCompletionCost(model, completionTks) + completionCost, err := ce.estimateCost("completion", modelWithCtx, completionTks) if err != nil { return 0, err } @@ -81,21 +111,23 @@ func (ce *CostEstimator) EstimateTotalCost(model string, promptTks, completionTk return promptCost + completionCost, nil } -func (ce *CostEstimator) EstimatePromptCost(model string, tks int) (float64, error) { - costMap, ok := ce.tokenCostMap["prompt"] - if !ok { - return 0, errors.New("prompt token cost is not provided") - +// normalizeAnthropicModel strips version/date suffixes (and Bedrock's +// cross-region "us.anthropic...." prefix) down to the base model family name +// used as a key in AnthropicPerMillionTokenCost. +func normalizeAnthropicModel(model string) string { + if strings.HasPrefix(model, "us") { + return convertAmazonModelToAnthropicModel(model) } + return SelectModel(model) +} - selected := "" - if strings.HasPrefix(model, "us") { - selected = convertAmazonModelToAnthropicModel(model) - } else { - selected = SelectModel(model) +func (ce *CostEstimator) estimateCost(costMapKey, model string, tks int) (float64, error) { + costMap, ok := ce.tokenCostMap[costMapKey] + if !ok { + return 0, fmt.Errorf("%s token cost is not provided", costMapKey) } - cost, ok := costMap[selected] + cost, ok := costMap[model] if !ok { return 0, fmt.Errorf("%s is not present in the cost map provided", model) } @@ -104,7 +136,30 @@ func (ce *CostEstimator) EstimatePromptCost(model string, tks int) (float64, err return tksInFloat / 1000000 * cost, nil } +func (ce *CostEstimator) EstimatePromptCost(model string, tks int) (float64, error) { + modelWithCtx := provider.ModelWithContextLength(normalizeAnthropicModel(model), int64(tks)) + return ce.estimateCost("prompt", modelWithCtx, tks) +} + func SelectModel(model string) string { + if strings.HasPrefix(model, "claude-mythos-5-1") { + return "claude-mythos-5-1" + } + if strings.HasPrefix(model, "claude-mythos-5") { + return "claude-mythos-5" + } + if strings.HasPrefix(model, "claude-fable-5-1") { + return "claude-fable-5-1" + } + if strings.HasPrefix(model, "claude-fable-5") { + return "claude-fable-5" + } + if strings.HasPrefix(model, "claude-opus-5-5") { + return "claude-opus-5-5" + } + if strings.HasPrefix(model, "claude-opus-5") { + return "claude-opus-5" + } if strings.HasPrefix(model, "claude-opus-4-8") { return "claude-opus-4-8" } @@ -123,6 +178,12 @@ func SelectModel(model string) string { if strings.HasPrefix(model, "claude-opus-4") { return "claude-opus-4" } + if strings.HasPrefix(model, "claude-sonnet-5-5") { + return "claude-sonnet-5-5" + } + if strings.HasPrefix(model, "claude-sonnet-5") { + return "claude-sonnet-5" + } if strings.HasPrefix(model, "claude-sonnet-4-6") { return "claude-sonnet-4-6" } @@ -132,6 +193,9 @@ func SelectModel(model string) string { if strings.HasPrefix(model, "claude-sonnet-4") { return "claude-sonnet-4" } + if strings.HasPrefix(model, "claude-haiku-5-5") { + return "claude-haiku-5-5" + } if strings.HasPrefix(model, "claude-haiku-4-5") { return "claude-haiku-4-5" } @@ -164,26 +228,9 @@ func convertAmazonModelToAnthropicModel(model string) string { return SelectModel(parts[2]) } -func (ce *CostEstimator) EstimateCompletionCost(model string, tks int) (float64, error) { - costMap, ok := ce.tokenCostMap["completion"] - if !ok { - return 0, errors.New("prompt token cost is not provided") - } - - selected := "" - if strings.HasPrefix(model, "us") { - selected = convertAmazonModelToAnthropicModel(model) - } else { - selected = SelectModel(model) - } - - cost, ok := costMap[selected] - if !ok { - return 0, errors.New("model is not present in the cost map provided") - } - - tksInFloat := float64(tks) - return tksInFloat / 1000000 * cost, nil +func (ce *CostEstimator) EstimateCompletionCost(model string, promptTks, completionTks int) (float64, error) { + modelWithCtx := provider.ModelWithContextLength(normalizeAnthropicModel(model), int64(promptTks)) + return ce.estimateCost("completion", modelWithCtx, completionTks) } func (ce *CostEstimator) Count(input string) int { diff --git a/internal/provider/openai/cost.go b/internal/provider/openai/cost.go index 93fd7ff..0b49542 100644 --- a/internal/provider/openai/cost.go +++ b/internal/provider/openai/cost.go @@ -8,6 +8,7 @@ import ( "slices" "strings" + "github.com/bricks-cloud/bricksllm/internal/provider" "github.com/bricks-cloud/bricksllm/internal/util" responsesOpenai "github.com/openai/openai-go/responses" goopenai "github.com/sashabaranov/go-openai" @@ -41,6 +42,21 @@ var OpenAiPerThousandTokenCost = map[string]map[string]float64{ "chatgpt-image-latest": 0.005, "gpt-image-1-mini": 0.002, + "gpt-6-astra": 0.020, + "gpt-6-astra~long": 0.020, + "gpt-6-astra~short": 0.010, + "gpt-6.1-sol": 0.004, + "gpt-6.1-sol~long": 0.004, + "gpt-6.1-sol~short": 0.002, + "gpt-6-luna": 0.0002, + "gpt-6-luna~long": 0.0002, + "gpt-6-luna~short": 0.0001, + + "gpt-5.6-sol": 0.008, + "gpt-5.6-sol~long": 0.008, + "gpt-5.6-sol~short": 0.004, + "gpt-5.6-cyber": 0.0125, + "gpt-5.5": 0.01, "gpt-5.5~long": 0.01, "gpt-5.5~short": 0.005, @@ -122,6 +138,21 @@ var OpenAiPerThousandTokenCost = map[string]map[string]float64{ "babbage-002": 0.000400, }, "cached-prompt": { + "gpt-6-astra": 0.002, + "gpt-6-astra~long": 0.002, + "gpt-6-astra~short": 0.001, + "gpt-6.1-sol": 0.0002, + "gpt-6.1-sol~long": 0.0002, + "gpt-6.1-sol~short": 0.0001, + "gpt-6-luna": 0.00002, + "gpt-6-luna~long": 0.00002, + "gpt-6-luna~short": 0.00001, + + "gpt-5.6-sol": 0.0008, + "gpt-5.6-sol~long": 0.0008, + "gpt-5.6-sol~short": 0.0004, + "gpt-5.6-cyber": 0.00125, + "gpt-5.5": 0.001, "gpt-5.5~long": 0.001, "gpt-5.5~short": 0.0005, @@ -213,6 +244,21 @@ var OpenAiPerThousandTokenCost = map[string]map[string]float64{ "gpt-image-1.5": 0.010, "chatgpt-image-latest": 0.010, + "gpt-6-astra": 0.075, + "gpt-6-astra~long": 0.075, + "gpt-6-astra~short": 0.05, + "gpt-6.1-sol": 0.015, + "gpt-6.1-sol~long": 0.015, + "gpt-6.1-sol~short": 0.01, + "gpt-6-luna": 0.00075, + "gpt-6-luna~long": 0.00075, + "gpt-6-luna~short": 0.0005, + + "gpt-5.6-sol": 0.03, + "gpt-5.6-sol~long": 0.03, + "gpt-5.6-sol~short": 0.02, + "gpt-5.6-cyber": 0.075, + "gpt-5.5": 0.045, "gpt-5.5~long": 0.045, "gpt-5.5~short": 0.03, @@ -425,7 +471,7 @@ func NewCostEstimator(m map[string]map[string]float64, tc tokenCounter) *CostEst func (ce *CostEstimator) EstimateTotalCost(model string, promptTks, completionTks int) (float64, error) { totalTokens := int64(promptTks + completionTks) - model = ModelWithContextLength(model, totalTokens) + model = provider.ModelWithContextLength(model, totalTokens) promptCost, err := ce.EstimatePromptCost(model, promptTks) if err != nil { return 0, err @@ -495,42 +541,34 @@ func (ce *CostEstimator) EstimateChatCompletionPromptTokenCounts(model string, r return tks, nil } -func (ce *CostEstimator) EstimateChatCompletionPromptCostWithTokenCounts(r *goopenai.ChatCompletionRequest) (int, float64, error) { - if len(r.Model) == 0 { - return 0, 0, errors.New("model is not provided") - } - - tks, err := countTotalTokens(r.Model, r, ce.tc) - if err != nil { - return 0, 0, err +// EstimateChatCompletionStreamCostWithTokenCounts estimates the cost of a +// streamed chat completion. promptTks must be the already-counted prompt +// token count so that the long/short context tier is picked using +// promptTks+completionTks, matching the tiering used by EstimateTotalCost +// for non-streaming requests. +func (ce *CostEstimator) EstimateChatCompletionStreamCostWithTokenCounts(model string, promptTks int, content string) (int, float64, float64, error) { + if len(model) == 0 { + return 0, 0, 0, errors.New("model is not provided") } - model := ModelWithContextLength(r.Model, int64(tks)) - cost, err := ce.EstimatePromptCost(model, tks) + tks, err := ce.tc.Count(model, content) if err != nil { - return 0, 0, err + return 0, 0, 0, err } - return tks, cost, nil -} + tieredModel := provider.ModelWithContextLength(model, int64(promptTks+tks)) -func (ce *CostEstimator) EstimateChatCompletionStreamCostWithTokenCounts(model string, content string) (int, float64, error) { - if len(model) == 0 { - return 0, 0, errors.New("model is not provided") - } - - tks, err := ce.tc.Count(model, content) + promptCost, err := ce.EstimatePromptCost(tieredModel, promptTks) if err != nil { - return 0, 0, err + return 0, 0, 0, err } - model = ModelWithContextLength(model, int64(tks)) - cost, err := ce.EstimateCompletionCost(model, tks) + completionCost, err := ce.EstimateCompletionCost(tieredModel, tks) if err != nil { - return 0, 0, err + return 0, 0, 0, err } - return tks, cost, nil + return tks, promptCost, completionCost, nil } func (ce *CostEstimator) EstimateCompletionsRequestCostWithTokenCounts(model string, content any) (int, float64, error) { @@ -850,7 +888,7 @@ func (ce *CostEstimator) EstimateResponseApiTotalCost(model string, usage respon outputTokens := usage.OutputTokens totalTokens := inputTokens + cachedInputTokens + outputTokens - model = ModelWithContextLength(model, totalTokens) + model = provider.ModelWithContextLength(model, totalTokens) cachedInputCost, err := ce.estimateResponseApiTokensCost("cached-prompt", model, cachedInputTokens) if err != nil { @@ -1110,25 +1148,3 @@ func countTotalTokens(model string, r *goopenai.ChatCompletionRequest, tc tokenC return tks + ftks + mtks, err } - -var modelWithLengthCtx = []string{ - "gpt-5.5", - "gpt-5.5-pro", - "gpt-5.4", - "gpt-5.4-pro", -} - -func ModelWithContextLength(model string, tokens int64) string { - trimmed := strings.TrimSpace(model) - if slices.Contains(modelWithLengthCtx, trimmed) { - return trimmed + contextLengthSuffixByTokens(tokens) - } - return trimmed -} - -func contextLengthSuffixByTokens(tokens int64) string { - if tokens >= 272000 { - return "~long" - } - return "~short" -} diff --git a/internal/provider/utils.go b/internal/provider/utils.go new file mode 100644 index 0000000..f4aedc6 --- /dev/null +++ b/internal/provider/utils.go @@ -0,0 +1,42 @@ +package provider + +import ( + "strings" +) + +type ModelContextLenght struct { + Model string + Tokens int64 +} + +var modelWithLengthCtx = []ModelContextLenght{ + // open ai + ModelContextLenght{Model: "gpt-6-astra", Tokens: 272000}, + ModelContextLenght{Model: "gpt-6.1-sol", Tokens: 272000}, + ModelContextLenght{Model: "gpt-6-luna", Tokens: 272000}, + ModelContextLenght{Model: "gpt-5.6-sol", Tokens: 272000}, + ModelContextLenght{Model: "gpt-5.5", Tokens: 272000}, + ModelContextLenght{Model: "gpt-5.5-pro", Tokens: 272000}, + ModelContextLenght{Model: "gpt-5.4", Tokens: 272000}, + ModelContextLenght{Model: "gpt-5.4-pro", Tokens: 272000}, + + // anthropic + ModelContextLenght{Model: "claude-haiku-5-5", Tokens: 100001}, +} + +func ModelWithContextLength(model string, tokens int64) string { + trueModel := strings.TrimSpace(model) + for _, m := range modelWithLengthCtx { + if trueModel == m.Model { + return trueModel + contextLengthSuffixByTokens(tokens, m.Tokens) + } + } + return trueModel +} + +func contextLengthSuffixByTokens(tokens, threshold int64) string { + if tokens >= threshold { + return "~long" + } + return "~short" +} diff --git a/internal/server/web/proxy/anthropic.go b/internal/server/web/proxy/anthropic.go index 79b479c..8c70d9e 100644 --- a/internal/server/web/proxy/anthropic.go +++ b/internal/server/web/proxy/anthropic.go @@ -22,7 +22,7 @@ import ( type anthropicEstimator interface { EstimateTotalCost(model string, promptTks, completionTks int) (float64, error) - EstimateCompletionCost(model string, tks int) (float64, error) + EstimateCompletionCost(model string, promptTks, completionTks int) (float64, error) EstimatePromptCost(model string, tks int) (float64, error) Count(input string) int CountMessagesTokens(messages []anthropic.Message) int @@ -425,7 +425,7 @@ func getMessagesHandler(prod, private bool, client http.Client, e anthropicEstim defer func() { tks := response.Usage.OutputTokens model := c.GetString("model") - cost, err := e.EstimateCompletionCost(model, tks) + cost, err := e.EstimateCompletionCost(model, response.Usage.InputTokens, tks) if err != nil { telemetry.Incr("bricksllm.proxy.get_messages_handler.estimate_messages_cost_error", nil, 1) logError(log, "error when estimating anthropic messages stream cost", prod, err) diff --git a/internal/server/web/proxy/bedrock.go b/internal/server/web/proxy/bedrock.go index d9dc977..b1b6f2e 100644 --- a/internal/server/web/proxy/bedrock.go +++ b/internal/server/web/proxy/bedrock.go @@ -162,7 +162,7 @@ func getBedrockCompletionHandler(prod bool, e anthropicEstimator) gin.HandlerFun defer func() { model := c.GetString("model") translatedModel := util.TranslateBedrockModelToAnthropicModel(model) - compeltionCost, err := e.EstimateCompletionCost(translatedModel, completionTokenCount) + compeltionCost, err := e.EstimateCompletionCost(translatedModel, promptTokenCount, completionTokenCount) if err != nil { telemetry.Incr("bricksllm.proxy.get_bedrock_completion_handler.estimate_completion_cost_error", nil, 1) logError(log, "error when estimating bedrock completion cost", prod, err) @@ -437,7 +437,7 @@ func getBedrockMessagesHandler(prod bool, e anthropicEstimator) gin.HandlerFunc defer func() { model := c.GetString("model") translatedModel := util.TranslateBedrockModelToAnthropicModel(model) - compeltionCost, err := e.EstimateCompletionCost(translatedModel, completionTokenCount) + compeltionCost, err := e.EstimateCompletionCost(translatedModel, promptTokenCount, completionTokenCount) if err != nil { telemetry.Incr("bricksllm.proxy.get_bedrock_messages_handler.estimate_completion_cost_error", nil, 1) logError(log, "error when estimating bedrock completion cost", prod, err) diff --git a/internal/server/web/proxy/middleware.go b/internal/server/web/proxy/middleware.go index a18565a..674497a 100644 --- a/internal/server/web/proxy/middleware.go +++ b/internal/server/web/proxy/middleware.go @@ -52,9 +52,8 @@ type keyStorage interface { type estimator interface { EstimateTranscriptionCost(secs float64, model string, usage *openai.TranscriptionResponseUsage) (float64, error) EstimateSpeechCost(input string, model string) (float64, error) - EstimateChatCompletionPromptCostWithTokenCounts(r *goopenai.ChatCompletionRequest) (int, float64, error) EstimateEmbeddingsCost(r *goopenai.EmbeddingRequest) (float64, error) - EstimateChatCompletionStreamCostWithTokenCounts(model, content string) (int, float64, error) + EstimateChatCompletionStreamCostWithTokenCounts(model string, promptTks int, content string) (int, float64, float64, error) EstimateCompletionCost(model string, tks int) (float64, error) EstimateTotalCost(model string, promptTks, completionTks int) (float64, error) EstimateEmbeddingsInputCost(model string, tks int) (float64, error)