diff --git a/docs/openapi/relay.json b/docs/openapi/relay.json index 2e04217e..30e9cbde 100644 --- a/docs/openapi/relay.json +++ b/docs/openapi/relay.json @@ -3981,7 +3981,8 @@ }, "model": { "type": "string", - "example": "text-moderation-latest" + "description": "Optional. Defaults to omni-moderation-latest when omitted. Tokens with model limits must explicitly allow this model. Channels serving model-less moderation requests must include omni-moderation-latest in their model list.", + "example": "omni-moderation-latest" } } }, diff --git a/middleware/distributor.go b/middleware/distributor.go index d99f15bb..53c45eea 100644 --- a/middleware/distributor.go +++ b/middleware/distributor.go @@ -388,7 +388,7 @@ func getModelRequest(c *gin.Context) (*ModelRequest, bool, error) { } if strings.HasPrefix(c.Request.URL.Path, "/v1/moderations") { if modelRequest.Model == "" { - modelRequest.Model = "text-moderation-stable" + modelRequest.Model = "omni-moderation-latest" } } if strings.HasSuffix(c.Request.URL.Path, "embeddings") { diff --git a/middleware/distributor_routing_test.go b/middleware/distributor_routing_test.go index 18c73687..5b4fd01b 100644 --- a/middleware/distributor_routing_test.go +++ b/middleware/distributor_routing_test.go @@ -44,6 +44,19 @@ func TestGetModelFromJSONBodyPreservesProviderRoutingFields(t *testing.T) { require.Equal(t, "openai", gjson.GetBytes(body, "provider.order.0").String()) } +func TestGetModelRequestDefaultsOmittedModerationModel(t *testing.T) { + gin.SetMode(gin.TestMode) + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + ctx.Request = httptest.NewRequest(http.MethodPost, "/v1/moderations", bytes.NewBufferString(`{"input":"check this"}`)) + ctx.Request.Header.Set("Content-Type", "application/json") + t.Cleanup(func() { common.CleanupBodyStorage(ctx) }) + + request, shouldSelectChannel, err := getModelRequest(ctx) + require.NoError(t, err) + require.True(t, shouldSelectChannel) + require.Equal(t, "omni-moderation-latest", request.Model) +} + func TestPlaygroundExplicitGroupBypassesStoredTokenRoutePlan(t *testing.T) { previousDB := model.DB previousMemoryCache := common.MemoryCacheEnabled diff --git a/relay/helper/moderation_request_test.go b/relay/helper/moderation_request_test.go new file mode 100644 index 00000000..0b704987 --- /dev/null +++ b/relay/helper/moderation_request_test.go @@ -0,0 +1,25 @@ +package helper + +import ( + "bytes" + "net/http" + "net/http/httptest" + "testing" + + "github.com/MAX-API-Next/MAX-API/common" + relayconstant "github.com/MAX-API-Next/MAX-API/relay/constant" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +func TestGetAndValidateTextRequestDefaultsOmittedModerationModel(t *testing.T) { + gin.SetMode(gin.TestMode) + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + ctx.Request = httptest.NewRequest(http.MethodPost, "/v1/moderations", bytes.NewBufferString(`{"input":"check this"}`)) + ctx.Request.Header.Set("Content-Type", "application/json") + t.Cleanup(func() { common.CleanupBodyStorage(ctx) }) + + request, err := GetAndValidateTextRequest(ctx, relayconstant.RelayModeModerations) + require.NoError(t, err) + require.Equal(t, "omni-moderation-latest", request.Model) +} diff --git a/relay/helper/price_test.go b/relay/helper/price_test.go index f9c2bcdc..3b150914 100644 --- a/relay/helper/price_test.go +++ b/relay/helper/price_test.go @@ -135,6 +135,42 @@ func TestModelPriceHelperDefersAlphaSearchFloorUntilSurcharge(t *testing.T) { require.Equal(t, 200, priceData.QuotaToPreConsume) } +func TestModelPriceHelperDoesNotReserveQuotaForOmniModeration(t *testing.T) { + gin.SetMode(gin.TestMode) + originalModelRatio := ratio_setting.ModelRatio2JSONString() + originalGroupRatio := ratio_setting.GroupRatio2JSONString() + originalPreConsumedQuota := common.PreConsumedQuota + originalFreeModelPreConsume := operation_setting.GetQuotaSetting().EnableFreeModelPreConsume + t.Cleanup(func() { + require.NoError(t, ratio_setting.UpdateModelRatioByJSONString(originalModelRatio)) + require.NoError(t, ratio_setting.UpdateGroupRatioByJSONString(originalGroupRatio)) + common.PreConsumedQuota = originalPreConsumedQuota + operation_setting.GetQuotaSetting().EnableFreeModelPreConsume = originalFreeModelPreConsume + }) + require.NoError(t, ratio_setting.UpdateModelRatioByJSONString(`{"omni-moderation-latest":0}`)) + require.NoError(t, ratio_setting.UpdateGroupRatioByJSONString(`{"default":1}`)) + operation_setting.GetQuotaSetting().EnableFreeModelPreConsume = false + + ctx, _ := gin.CreateTestContext(httptest.NewRecorder()) + info := &relaycommon.RelayInfo{ + RelayMode: relayconstant.RelayModeModerations, + OriginModelName: "omni-moderation-latest", + UserGroup: "default", + UsingGroup: "default", + } + + priceData, err := ModelPriceHelper(ctx, info, 100, &types.TokenCountMeta{}) + require.NoError(t, err) + require.Zero(t, priceData.ModelRatio) + require.Zero(t, priceData.QuotaToPreConsume) + + common.PreConsumedQuota = 500 + operation_setting.GetQuotaSetting().EnableFreeModelPreConsume = true + priceData, err = ModelPriceHelper(ctx, info, 100, &types.TokenCountMeta{}) + require.NoError(t, err) + require.Equal(t, 500, priceData.QuotaToPreConsume) +} + func TestModelPriceHelperPerCallRejectsUnpricedMJSunoTaskModels(t *testing.T) { gin.SetMode(gin.TestMode) originalSelfUseMode := operation_setting.SelfUseModeEnabled diff --git a/relay/helper/valid_request.go b/relay/helper/valid_request.go index 567f8bec..7cf99f43 100644 --- a/relay/helper/valid_request.go +++ b/relay/helper/valid_request.go @@ -309,7 +309,7 @@ func GetAndValidateTextRequest(c *gin.Context, relayMode int) (*dto.GeneralOpenA } if relayMode == relayconstant.RelayModeModerations && textRequest.Model == "" { - textRequest.Model = "text-moderation-latest" + textRequest.Model = "omni-moderation-latest" } if relayMode == relayconstant.RelayModeEmbeddings && textRequest.Model == "" { textRequest.Model = c.Param("model") diff --git a/setting/ratio_setting/model_ratio.go b/setting/ratio_setting/model_ratio.go index dd88da35..ec2c1914 100644 --- a/setting/ratio_setting/model_ratio.go +++ b/setting/ratio_setting/model_ratio.go @@ -130,6 +130,8 @@ var defaultModelRatio = map[string]float64{ "text-search-ada-doc-001": 10, "text-moderation-stable": 0.1, "text-moderation-latest": 0.1, + "omni-moderation-latest": 0, // OpenAI moderation models are free + "omni-moderation-2024-09-26": 0, "claude-3-haiku-20240307": 0.125, // $0.25 / 1M tokens "claude-3-5-haiku-20241022": 0.5, // $1 / 1M tokens "claude-haiku-4-5-20251001": 0.5, // $1 / 1M tokens diff --git a/setting/ratio_setting/pricing_validation_test.go b/setting/ratio_setting/pricing_validation_test.go index ba1925dc..5730eaac 100644 --- a/setting/ratio_setting/pricing_validation_test.go +++ b/setting/ratio_setting/pricing_validation_test.go @@ -48,6 +48,14 @@ func TestLoadPricingMapWithoutNormalizerPreservesRawKeys(t *testing.T) { require.Equal(t, 2.5, value) } +func TestDefaultModerationRatiosAreFree(t *testing.T) { + for _, model := range []string{"omni-moderation-latest", "omni-moderation-2024-09-26"} { + ratio, ok := GetDefaultModelRatioMap()[model] + require.True(t, ok, model) + require.Zero(t, ratio, model) + } +} + func TestValidatePricingMapRejectsOversizedJSONBeforeLoad(t *testing.T) { raw := `{"` + strings.Repeat("a", maxPricingMapJSONBytes) + `":1}`