From 900dd3b31235616cae6586164140105c27fdbdf0 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Mon, 31 Aug 2026 02:09:53 +0800 Subject: [PATCH 01/19] fix bugs --- controller/audit.go | 2 + controller/smart_ops_alert.go | 116 +++ controller/smart_ops_alert_test.go | 21 + model/billing_settlement.go | 252 +++++- model/billing_settlement_regression_test.go | 154 +++- router/api-router.go | 3 + service/billing_session.go | 38 +- service/billing_session_test.go | 15 + service/smart_ops_alert.go | 80 +- service/smart_ops_alert_test.go | 41 + .../billing_reconciliation_setting/config.go | 39 + .../config_test.go | 15 + .../src/features/smart-ops/alerts.test.tsx | 339 +++++++- web/default/src/features/smart-ops/alerts.tsx | 809 ++++++++++++++++-- web/default/src/features/smart-ops/api.ts | 33 + web/default/src/features/smart-ops/types.ts | 57 ++ web/default/src/i18n/locales/en.json | 30 + web/default/src/i18n/locales/fr.json | 30 + web/default/src/i18n/locales/ja.json | 30 + web/default/src/i18n/locales/ru.json | 30 + web/default/src/i18n/locales/vi.json | 30 + web/default/src/i18n/locales/zh.json | 30 + 22 files changed, 2101 insertions(+), 93 deletions(-) create mode 100644 controller/smart_ops_alert_test.go create mode 100644 setting/billing_reconciliation_setting/config.go create mode 100644 setting/billing_reconciliation_setting/config_test.go diff --git a/controller/audit.go b/controller/audit.go index fac35107..bf488d84 100644 --- a/controller/audit.go +++ b/controller/audit.go @@ -31,6 +31,8 @@ var auditContentTemplates = map[string]string{ "user.topup_complete": "Completed top-up order ${tradeNo}", "option.update": "Updated system setting ${key}", "option.reset_ratio": "Reset model ratio settings", + "billing.reconciliation_blocking_policy_update": "Updated billing reconciliation default user-blocking policy to ${block_user_by_default}", + "billing.reconciliation_review": "Reviewed billing settlement ${settlement_id} (block user: ${block_user})", "channel.create": "Created channel ${name} (type ${type}, count ${count})", "channel.update": "Updated channel ${name} (ID: ${id})", diff --git a/controller/smart_ops_alert.go b/controller/smart_ops_alert.go index db14a395..bd2370c9 100644 --- a/controller/smart_ops_alert.go +++ b/controller/smart_ops_alert.go @@ -1,8 +1,12 @@ package controller import ( + "errors" "net/http" + "strconv" + "github.com/MAX-API-Next/MAX-API/common" + "github.com/MAX-API-Next/MAX-API/model" "github.com/MAX-API-Next/MAX-API/service" "github.com/gin-gonic/gin" ) @@ -15,3 +19,115 @@ func GetSmartOpsAlerts(c *gin.Context) { "data": service.GetSmartOpsAlerts(), }) } + +type billingSettlementBlockingPolicyRequest struct { + BlockUserByDefault *bool `json:"block_user_by_default"` +} + +func UpdateBillingSettlementBlockingPolicy(c *gin.Context) { + var request billingSettlementBlockingPolicyRequest + if err := common.DecodeJson(c.Request.Body, &request); err != nil || request.BlockUserByDefault == nil { + c.JSON(http.StatusBadRequest, gin.H{ + "success": false, + "message": "block_user_by_default is required", + }) + return + } + if err := service.UpdateBillingSettlementBlockingPolicy(*request.BlockUserByDefault); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{ + "success": false, + "message": "failed to update billing reconciliation blocking policy", + }) + return + } + recordManageAudit(c, "billing.reconciliation_blocking_policy_update", map[string]interface{}{ + "block_user_by_default": *request.BlockUserByDefault, + }) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": gin.H{ + "block_user_by_default": *request.BlockUserByDefault, + }, + }) +} + +type billingSettlementReviewRequest struct { + BlockUser *bool `json:"block_user"` + Note string `json:"note"` +} + +func ReviewBillingSettlement(c *gin.Context) { + id, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil || id <= 0 { + c.JSON(http.StatusBadRequest, gin.H{ + "success": false, + "message": "invalid billing settlement id", + }) + return + } + var request billingSettlementReviewRequest + if err := common.DecodeJson(c.Request.Body, &request); err != nil { + c.JSON(http.StatusBadRequest, gin.H{ + "success": false, + "message": "invalid billing settlement review request", + }) + return + } + record, err := service.ReviewBillingSettlement(id, c.GetInt("id"), request.BlockUser, request.Note) + if err != nil { + status := http.StatusInternalServerError + message := "failed to review billing settlement" + switch { + case errors.Is(err, service.ErrInvalidBillingSettlementReconciliationReview): + status = http.StatusBadRequest + message = err.Error() + case errors.Is(err, model.ErrBillingSettlementReviewConflict): + status = http.StatusConflict + message = "billing settlement is no longer pending manual reconciliation" + } + c.JSON(status, gin.H{"success": false, "message": message}) + return + } + recordManageAuditFor(c, record.UserID, "billing.reconciliation_review", map[string]interface{}{ + "settlement_id": record.ID, + "block_user": *request.BlockUser, + }) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": gin.H{ + "id": record.ID, + "reconciliation_reviewed_at": record.ReconciliationReviewedAt, + "block_user": *request.BlockUser, + }, + }) +} + +// GetBillingSettlementReconciliation exposes bounded, read-only evidence for +// unresolved positive final settlements. It does not retry or mutate records. +func GetBillingSettlementReconciliation(c *gin.Context) { + limit, err := parseIntQuery(c, "limit") + if err != nil { + respondInvalidPerformanceQuery( + c, + err, + service.ErrInvalidBillingSettlementReconciliationQuery, + "failed to query billing settlement reconciliation", + ) + return + } + result, err := service.GetBillingSettlementReconciliation(limit) + if err != nil { + respondPerformanceError( + c, + err, + service.ErrInvalidBillingSettlementReconciliationQuery, + "failed to query billing settlement reconciliation", + ) + return + } + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": result, + }) +} diff --git a/controller/smart_ops_alert_test.go b/controller/smart_ops_alert_test.go new file mode 100644 index 00000000..e5fa249d --- /dev/null +++ b/controller/smart_ops_alert_test.go @@ -0,0 +1,21 @@ +package controller + +import ( + "testing" + + "github.com/MAX-API-Next/MAX-API/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBillingSettlementMutationRequestsPreserveExplicitFalse(t *testing.T) { + var policy billingSettlementBlockingPolicyRequest + require.NoError(t, common.Unmarshal([]byte(`{"block_user_by_default":false}`), &policy)) + require.NotNil(t, policy.BlockUserByDefault) + assert.False(t, *policy.BlockUserByDefault) + + var review billingSettlementReviewRequest + require.NoError(t, common.Unmarshal([]byte(`{"block_user":false,"note":"verified"}`), &review)) + require.NotNil(t, review.BlockUser) + assert.False(t, *review.BlockUser) +} diff --git a/model/billing_settlement.go b/model/billing_settlement.go index 51907c07..ad095197 100644 --- a/model/billing_settlement.go +++ b/model/billing_settlement.go @@ -2,12 +2,14 @@ package model import ( "context" + "database/sql" "errors" "fmt" "sync" "time" "github.com/MAX-API-Next/MAX-API/common" + "github.com/MAX-API-Next/MAX-API/setting/billing_reconciliation_setting" "github.com/bytedance/gopkg/util/gopool" "gorm.io/gorm" "gorm.io/gorm/clause" @@ -34,6 +36,7 @@ var ( ErrBillingSettlementTaskConflict = errors.New("billing settlement task quota conflict") ErrBillingSettlementOperationConflict = errors.New("billing settlement operation conflict") ErrBillingSettlementRecordNotDurable = errors.New("billing settlement record was not durably created") + ErrBillingSettlementReviewConflict = errors.New("billing settlement is no longer reviewable") ErrSubscriptionSettlementUnbound = errors.New("subscription settlement is not bound to its pre-consume request") ErrSubscriptionSettlementPeriodChanged = errors.New("subscription settlement crossed a quota reset period") ) @@ -70,14 +73,14 @@ type BillingSettlement struct { UserID int `gorm:"not null;default:0;index:idx_billing_settlement_admission,priority:1"` SubscriptionID int `gorm:"not null;default:0"` TokenID int `gorm:"not null;default:0"` - FundingDelta int64 `gorm:"not null;default:0;index:idx_billing_settlement_admission,priority:3;index:idx_billing_settlement_status_funding,priority:2"` + FundingDelta int64 `gorm:"not null;default:0;index:idx_billing_settlement_admission,priority:3;index:idx_billing_settlement_status_funding,priority:2;index:idx_billing_settlement_reconciliation,priority:2"` AppliedFundingDelta int64 `gorm:"not null;default:0"` TokenDelta int64 `gorm:"not null;default:0"` AppliedTokenDelta int64 `gorm:"not null;default:0"` TaskID int64 `gorm:"not null;default:0"` TaskQuota int64 `gorm:"not null;default:0"` TaskQuotaTarget int64 `gorm:"not null;default:0"` - Status string `gorm:"type:varchar(16);index;index:idx_billing_settlement_admission,priority:2;index:idx_billing_settlement_status_funding,priority:1;not null;default:'applied'"` + Status string `gorm:"type:varchar(16);index;index:idx_billing_settlement_admission,priority:2;index:idx_billing_settlement_status_funding,priority:1;index:idx_billing_settlement_reconciliation,priority:1;not null;default:'applied'"` Attempts int `gorm:"not null;default:0"` LastError string `gorm:"type:text"` NextAttempt int64 `gorm:"index;not null;default:0"` @@ -92,6 +95,10 @@ type BillingSettlement struct { PreConsumeEffectiveQuota int64 `gorm:"not null;default:0"` EffectPayload string `gorm:"type:text"` EffectStatus string `gorm:"type:varchar(16);index"` + ReconciliationReviewedAt int64 `gorm:"not null;default:0;index:idx_billing_settlement_reconciliation,priority:3"` + ReconciliationReviewedBy int `gorm:"not null;default:0"` + ReconciliationReviewNote string `gorm:"type:text"` + UserBlockingOverride *bool `gorm:"index:idx_billing_settlement_reconciliation,priority:4"` CreatedAt int64 `gorm:"not null"` UpdatedAt int64 `gorm:"index;not null"` } @@ -165,13 +172,57 @@ type BillingSettlementInput struct { PreConsumeEffectiveQuota int64 } -// BillingSettlementBacklogStats is a read-only operational projection of -// positive final settlements that currently block new paid requests. +// BillingSettlementBacklogStats is a read-only operational projection of open +// positive-final-settlement alerts. User admission is evaluated separately by +// the global and per-record blocking policy. type BillingSettlementBacklogStats struct { Count int64 `gorm:"column:record_count"` OldestCreatedAt int64 `gorm:"column:oldest_created_at"` } +// BillingSettlementReconciliationItem is a read-only operator projection. It +// deliberately excludes token keys, effect payloads, request bodies, and any +// other secret-bearing fields. +type BillingSettlementReconciliationItem struct { + ID int64 `json:"id"` + OperationKey string `json:"operation_key"` + Status string `json:"status"` + Source string `json:"source"` + UserID int `json:"user_id"` + SubscriptionID int `json:"subscription_id"` + TokenID int `json:"token_id"` + TaskID int64 `json:"task_id"` + FundingDelta int64 `json:"funding_delta"` + AppliedFundingDelta int64 `json:"applied_funding_delta"` + TokenDelta int64 `json:"token_delta"` + AppliedTokenDelta int64 `json:"applied_token_delta"` + Attempts int `json:"attempts"` + LastError string `json:"last_error"` + NextAttempt int64 `json:"next_attempt"` + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` + ReconciliationReviewedAt int64 `json:"reconciliation_reviewed_at"` + ReconciliationReviewedBy int `json:"reconciliation_reviewed_by"` + ReconciliationReviewNote string `json:"reconciliation_review_note"` + UserBlockingOverride *bool `json:"user_blocking_override"` + BlocksUser bool `json:"blocks_user"` +} + +type BillingSettlementReconciliationData struct { + TotalCount int64 `json:"total_count"` + PendingCount int64 `json:"pending_count"` + ManualCount int64 `json:"manual_count"` + OpenAlertCount int64 `json:"open_alert_count"` + ReviewedCount int64 `json:"reviewed_count"` + BlockingRecordCount int64 `json:"blocking_record_count"` + BlockedUserCount int64 `json:"blocked_user_count"` + BlockUserByDefault bool `json:"block_user_by_default"` + OldestCreatedAt int64 `json:"oldest_created_at"` + Truncated bool `json:"truncated"` + GeneratedAt int64 `json:"generated_at"` + Items []BillingSettlementReconciliationItem `json:"items"` +} + func unresolvedPositiveFinalizeSettlementScope(db *gorm.DB) *gorm.DB { return db.Model(&BillingSettlement{}). Where("funding_delta > 0"). @@ -179,6 +230,19 @@ func unresolvedPositiveFinalizeSettlementScope(db *gorm.DB) *gorm.DB { Where("operation_key LIKE ?", BillingRequestFinalizeOperationKey("%")) } +func openPositiveFinalizeSettlementAlertScope(db *gorm.DB) *gorm.DB { + return unresolvedPositiveFinalizeSettlementScope(db). + Where("reconciliation_reviewed_at = ?", 0) +} + +func blockingPositiveFinalizeSettlementScope(db *gorm.DB, blockUserByDefault bool) *gorm.DB { + scope := unresolvedPositiveFinalizeSettlementScope(db) + if blockUserByDefault { + return scope.Where("user_blocking_override IS NULL OR user_blocking_override = ?", true) + } + return scope.Where("user_blocking_override = ?", true) +} + var billingSettlementRunnerOnce sync.Once var billingSettlementRunnerState struct { sync.Mutex @@ -271,21 +335,143 @@ func observeBillingSettlementBacklog(observedAt time.Time) { } } -// GetUnresolvedPositiveFinalizeSettlementStats returns the global count and -// oldest creation time for positive request-finalize settlements in pending or -// manual state. It is intentionally read-only and portable across all supported -// databases. +// ObserveBillingSettlementBacklogNow refreshes the read-only operational +// projection after an administrator review. It never retries or mutates a +// settlement or any financial balance. +func ObserveBillingSettlementBacklogNow() { + observeBillingSettlementBacklog(time.Now()) +} + +// GetUnresolvedPositiveFinalizeSettlementStats returns the count and oldest +// creation time for open (not yet reviewed) positive request-finalize alerts. +// Reviewed rows remain unresolved financial records and are still returned by +// the reconciliation list until the settlement itself reaches applied. func GetUnresolvedPositiveFinalizeSettlementStats() (BillingSettlementBacklogStats, error) { if DB == nil { return BillingSettlementBacklogStats{}, errors.New("database is not initialized") } + return getOpenPositiveFinalizeSettlementAlertStatsDB(DB) +} + +// GetUnresolvedPositiveFinalizeSettlements returns bounded, read-only evidence +// for operators. It never retries, updates, deletes, or otherwise mutates a +// settlement record or any financial balance. +func GetUnresolvedPositiveFinalizeSettlements(limit int) (BillingSettlementReconciliationData, error) { + if DB == nil { + return BillingSettlementReconciliationData{}, errors.New("database is not initialized") + } + if limit <= 0 { + limit = 100 + } + if limit > 200 { + limit = 200 + } + + data := BillingSettlementReconciliationData{ + BlockUserByDefault: billing_reconciliation_setting.BlockUserByDefault(), + GeneratedAt: time.Now().Unix(), + Items: make([]BillingSettlementReconciliationItem, 0), + } + transactionOptions := make([]*sql.TxOptions, 0, 1) + if common.UsingMySQL || common.UsingPostgreSQL { + transactionOptions = append(transactionOptions, &sql.TxOptions{ + Isolation: sql.LevelRepeatableRead, + ReadOnly: true, + }) + } + err := DB.Transaction(func(tx *gorm.DB) error { + stats, err := getAllUnresolvedPositiveFinalizeSettlementStatsDB(tx) + if err != nil { + return err + } + data.TotalCount = stats.Count + data.OldestCreatedAt = stats.OldestCreatedAt + + if err := unresolvedPositiveFinalizeSettlementScope(tx). + Where("status = ?", BillingSettlementStatusPending). + Count(&data.PendingCount).Error; err != nil { + return err + } + if err := unresolvedPositiveFinalizeSettlementScope(tx). + Where("status = ?", BillingSettlementStatusManual). + Count(&data.ManualCount).Error; err != nil { + return err + } + if err := openPositiveFinalizeSettlementAlertScope(tx). + Count(&data.OpenAlertCount).Error; err != nil { + return err + } + data.ReviewedCount = data.TotalCount - data.OpenAlertCount + blockingScope := blockingPositiveFinalizeSettlementScope(tx, data.BlockUserByDefault) + if err := blockingScope.Count(&data.BlockingRecordCount).Error; err != nil { + return err + } + if err := blockingPositiveFinalizeSettlementScope(tx, data.BlockUserByDefault). + Distinct("user_id"). + Count(&data.BlockedUserCount).Error; err != nil { + return err + } + + items := make([]BillingSettlementReconciliationItem, 0, limit+1) + if err := unresolvedPositiveFinalizeSettlementScope(tx). + Select( + "id", "operation_key", "status", "source", "user_id", "subscription_id", "token_id", "task_id", + "funding_delta", "applied_funding_delta", "token_delta", "applied_token_delta", "attempts", "last_error", + "next_attempt", "created_at", "updated_at", "reconciliation_reviewed_at", "reconciliation_reviewed_by", + "reconciliation_review_note", "user_blocking_override", + ). + Order("reconciliation_reviewed_at ASC"). + Order("created_at ASC"). + Order("id ASC"). + Limit(limit + 1). + Scan(&items).Error; err != nil { + return err + } + if len(items) > limit { + data.Truncated = true + items = items[:limit] + } + for index := range items { + items[index].LastError = common.SanitizePersistedLogContent( + common.MaskSensitiveInfo(items[index].LastError), + ) + items[index].ReconciliationReviewNote = common.SanitizePersistedLogContent( + common.MaskSensitiveInfo(items[index].ReconciliationReviewNote), + ) + items[index].BlocksUser = billingSettlementBlocksUser( + items[index].UserBlockingOverride, + data.BlockUserByDefault, + ) + } + data.Items = items + return nil + }, transactionOptions...) + return data, err +} + +func getAllUnresolvedPositiveFinalizeSettlementStatsDB(db *gorm.DB) (BillingSettlementBacklogStats, error) { + var stats BillingSettlementBacklogStats + err := unresolvedPositiveFinalizeSettlementScope(db). + Select("COUNT(*) AS record_count, COALESCE(MIN(created_at), 0) AS oldest_created_at"). + Scan(&stats).Error + return stats, err +} + +func getOpenPositiveFinalizeSettlementAlertStatsDB(db *gorm.DB) (BillingSettlementBacklogStats, error) { var stats BillingSettlementBacklogStats - err := unresolvedPositiveFinalizeSettlementScope(DB). + err := openPositiveFinalizeSettlementAlertScope(db). Select("COUNT(*) AS record_count, COALESCE(MIN(created_at), 0) AS oldest_created_at"). Scan(&stats).Error return stats, err } +func billingSettlementBlocksUser(override *bool, blockUserByDefault bool) bool { + if override != nil { + return *override + } + return blockUserByDefault +} + // HasUnresolvedPositiveFinalizeSettlement reports whether a user has an // upstream-accepted request whose positive final settlement has not completed. // New paid requests must stop before they can consume more service against the @@ -299,8 +485,9 @@ func HasUnresolvedPositiveFinalizeSettlement(userID int) (bool, error) { return false, nil } + blockUserByDefault := billing_reconciliation_setting.BlockUserByDefault() var record BillingSettlement - result := unresolvedPositiveFinalizeSettlementScope(DB). + result := blockingPositiveFinalizeSettlementScope(DB, blockUserByDefault). Select("id"). Where("user_id = ?", userID). Limit(1). @@ -311,6 +498,51 @@ func HasUnresolvedPositiveFinalizeSettlement(userID int) (bool, error) { return result.RowsAffected > 0, nil } +func billingSettlementReviewUpdates(reviewerID int, blockUser bool, note string, reviewedAt time.Time) map[string]interface{} { + return map[string]interface{}{ + "reconciliation_reviewed_at": reviewedAt.Unix(), + "reconciliation_reviewed_by": reviewerID, + "reconciliation_review_note": note, + "user_blocking_override": blockUser, + "updated_at": reviewedAt.Unix(), + "revision": gorm.Expr("revision + ?", 1), + } +} + +// ReviewBillingSettlement records an administrator's alert disposition without +// changing settlement status, balances, applied deltas, or effect state. +func ReviewBillingSettlement(id int64, reviewerID int, blockUser bool, note string) (BillingSettlement, error) { + if DB == nil { + return BillingSettlement{}, errors.New("database is not initialized") + } + if id <= 0 || reviewerID <= 0 { + return BillingSettlement{}, ErrBillingSettlementReviewConflict + } + + var record BillingSettlement + if err := DB.Select("id", "user_id").First(&record, id).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return BillingSettlement{}, ErrBillingSettlementReviewConflict + } + return BillingSettlement{}, err + } + + reviewedAt := time.Now() + result := unresolvedPositiveFinalizeSettlementScope(DB). + Where("id = ?", id). + Updates(billingSettlementReviewUpdates(reviewerID, blockUser, note, reviewedAt)) + if result.Error != nil { + return BillingSettlement{}, result.Error + } + if result.RowsAffected != 1 { + return BillingSettlement{}, ErrBillingSettlementReviewConflict + } + if err := DB.First(&record, id).Error; err != nil { + return BillingSettlement{}, err + } + return record, nil +} + func BillingRequestFinalizeOperationKey(requestID string) string { return billingRequestOperationPrefix + requestID + billingRequestFinalizeSuffix } diff --git a/model/billing_settlement_regression_test.go b/model/billing_settlement_regression_test.go index b4a89b20..9188df93 100644 --- a/model/billing_settlement_regression_test.go +++ b/model/billing_settlement_regression_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/MAX-API-Next/MAX-API/common" + "github.com/MAX-API-Next/MAX-API/setting/billing_reconciliation_setting" "github.com/glebarez/sqlite" "github.com/go-webauthn/webauthn/protocol" "github.com/go-webauthn/webauthn/webauthn" @@ -84,8 +85,10 @@ func TestConcurrentBillingPreConsumeSelectionAllowsOnlyOneFundingSource(t *testi func TestHasUnresolvedPositiveFinalizeSettlement(t *testing.T) { setupUserUpdateTestState(t) + setBillingReconciliationBlockDefaultForTest(t, true) require.True(t, DB.Migrator().HasIndex(&BillingSettlement{}, "idx_billing_settlement_admission")) require.True(t, DB.Migrator().HasIndex(&BillingSettlement{}, "idx_billing_settlement_status_funding")) + require.True(t, DB.Migrator().HasIndex(&BillingSettlement{}, "idx_billing_settlement_reconciliation")) const blockingUserID = 941 const nonBlockingUserID = 942 @@ -95,7 +98,8 @@ func TestHasUnresolvedPositiveFinalizeSettlement(t *testing.T) { { OperationKey: "request:pending-finalize:finalize", Source: BillingSettlementSourceWallet, UserID: blockingUserID, FundingDelta: 10, TokenDelta: 10, - Status: BillingSettlementStatusPending, NextAttempt: now, CreatedAt: oldest, UpdatedAt: now, Revision: 1, + Status: BillingSettlementStatusPending, LastError: "upstream https://api.example.com/v1?api_key=secret failed", + NextAttempt: now, CreatedAt: oldest, UpdatedAt: now, Revision: 1, }, { OperationKey: "request:manual-finalize:finalize", Source: BillingSettlementSourceWallet, @@ -129,6 +133,26 @@ func TestHasUnresolvedPositiveFinalizeSettlement(t *testing.T) { assert.EqualValues(t, 2, stats.Count) assert.Equal(t, oldest, stats.OldestCreatedAt) + reconciliation, err := GetUnresolvedPositiveFinalizeSettlements(1) + require.NoError(t, err) + assert.EqualValues(t, 2, reconciliation.TotalCount) + assert.EqualValues(t, 1, reconciliation.PendingCount) + assert.EqualValues(t, 1, reconciliation.ManualCount) + assert.EqualValues(t, 2, reconciliation.OpenAlertCount) + assert.Zero(t, reconciliation.ReviewedCount) + assert.EqualValues(t, 2, reconciliation.BlockingRecordCount) + assert.EqualValues(t, 1, reconciliation.BlockedUserCount) + assert.True(t, reconciliation.BlockUserByDefault) + assert.Equal(t, oldest, reconciliation.OldestCreatedAt) + assert.True(t, reconciliation.Truncated) + require.Len(t, reconciliation.Items, 1) + assert.Equal(t, "request:pending-finalize:finalize", reconciliation.Items[0].OperationKey) + assert.Equal(t, BillingSettlementStatusPending, reconciliation.Items[0].Status) + assert.EqualValues(t, 10, reconciliation.Items[0].FundingDelta) + assert.NotContains(t, reconciliation.Items[0].LastError, "api.example.com") + assert.NotContains(t, reconciliation.Items[0].LastError, "secret") + assert.Contains(t, reconciliation.Items[0].LastError, "***") + require.NoError(t, DB.Where("user_id = ?", blockingUserID).Delete(&BillingSettlement{}).Error) stats, err = GetUnresolvedPositiveFinalizeSettlementStats() require.NoError(t, err) @@ -136,6 +160,134 @@ func TestHasUnresolvedPositiveFinalizeSettlement(t *testing.T) { assert.Zero(t, stats.OldestCreatedAt) } +func TestBillingSettlementBlockingPolicyAndReviewAreIndependentFromFinancialState(t *testing.T) { + setupUserUpdateTestState(t) + setBillingReconciliationBlockDefaultForTest(t, false) + + now := time.Now().Unix() + block := true + allow := false + records := []BillingSettlement{ + { + OperationKey: "request:inherit-policy:finalize", Source: BillingSettlementSourceWallet, + UserID: 951, FundingDelta: 10, AppliedFundingDelta: 2, TokenDelta: 10, AppliedTokenDelta: 2, + Status: BillingSettlementStatusPending, CreatedAt: now - 30, UpdatedAt: now, Revision: 1, + }, + { + OperationKey: "request:force-block:finalize", Source: BillingSettlementSourceWallet, + UserID: 952, FundingDelta: 11, TokenDelta: 11, Status: BillingSettlementStatusManual, + UserBlockingOverride: &block, CreatedAt: now - 20, UpdatedAt: now, Revision: 1, + }, + { + OperationKey: "request:force-allow:finalize", Source: BillingSettlementSourceWallet, + UserID: 953, FundingDelta: 12, TokenDelta: 12, Status: BillingSettlementStatusPending, + UserBlockingOverride: &allow, CreatedAt: now - 10, UpdatedAt: now, Revision: 1, + }, + } + require.NoError(t, DB.Create(&records).Error) + + blocked, err := HasUnresolvedPositiveFinalizeSettlement(951) + require.NoError(t, err) + assert.False(t, blocked, "nil override must inherit the disabled global policy") + blocked, err = HasUnresolvedPositiveFinalizeSettlement(952) + require.NoError(t, err) + assert.True(t, blocked, "an explicit per-record block must override the global policy") + blocked, err = HasUnresolvedPositiveFinalizeSettlement(953) + require.NoError(t, err) + assert.False(t, blocked, "an explicit per-record allow must remain non-blocking") + + reviewed, err := ReviewBillingSettlement(records[0].ID, 7001, false, "Verified provider usage against the invoice") + require.NoError(t, err) + assert.Equal(t, records[0].ID, reviewed.ID) + assert.Equal(t, records[0].UserID, reviewed.UserID) + assert.Equal(t, BillingSettlementStatusPending, reviewed.Status) + assert.EqualValues(t, 10, reviewed.FundingDelta) + assert.EqualValues(t, 2, reviewed.AppliedFundingDelta) + assert.EqualValues(t, 10, reviewed.TokenDelta) + assert.EqualValues(t, 2, reviewed.AppliedTokenDelta) + assert.Positive(t, reviewed.ReconciliationReviewedAt) + assert.Equal(t, 7001, reviewed.ReconciliationReviewedBy) + assert.Equal(t, "Verified provider usage against the invoice", reviewed.ReconciliationReviewNote) + require.NotNil(t, reviewed.UserBlockingOverride) + assert.False(t, *reviewed.UserBlockingOverride, "explicit false must be durably persisted") + + stats, err := GetUnresolvedPositiveFinalizeSettlementStats() + require.NoError(t, err) + assert.EqualValues(t, 2, stats.Count, "reviewed records must leave the open-alert projection") + + reconciliation, err := GetUnresolvedPositiveFinalizeSettlements(100) + require.NoError(t, err) + assert.EqualValues(t, 3, reconciliation.TotalCount, "review does not complete or delete a settlement") + assert.EqualValues(t, 2, reconciliation.OpenAlertCount) + assert.EqualValues(t, 1, reconciliation.ReviewedCount) + assert.EqualValues(t, 1, reconciliation.BlockingRecordCount) + assert.EqualValues(t, 1, reconciliation.BlockedUserCount) + assert.False(t, reconciliation.BlockUserByDefault) + require.Len(t, reconciliation.Items, 3) + + var reviewedItem *BillingSettlementReconciliationItem + for index := range reconciliation.Items { + if reconciliation.Items[index].ID == records[0].ID { + reviewedItem = &reconciliation.Items[index] + break + } + } + require.NotNil(t, reviewedItem) + assert.False(t, reviewedItem.BlocksUser) + assert.Equal(t, 7001, reviewedItem.ReconciliationReviewedBy) + assert.Equal(t, "Verified provider usage against the invoice", reviewedItem.ReconciliationReviewNote) +} + +func TestReviewBillingSettlementRejectsAlreadyAppliedRecord(t *testing.T) { + setupUserUpdateTestState(t) + now := time.Now().Unix() + record := BillingSettlement{ + OperationKey: "request:review-race:finalize", Source: BillingSettlementSourceWallet, + UserID: 954, FundingDelta: 10, AppliedFundingDelta: 10, TokenDelta: 10, AppliedTokenDelta: 10, + Status: BillingSettlementStatusApplied, CreatedAt: now, UpdatedAt: now, Revision: 2, + } + require.NoError(t, DB.Create(&record).Error) + + _, err := ReviewBillingSettlement(record.ID, 7002, false, "Already reconciled elsewhere") + + assert.ErrorIs(t, err, ErrBillingSettlementReviewConflict) + var stored BillingSettlement + require.NoError(t, DB.First(&stored, record.ID).Error) + assert.Zero(t, stored.ReconciliationReviewedAt) + assert.Empty(t, stored.ReconciliationReviewNote) +} + +func TestBillingSettlementReviewUpdateMapUsesSchemaColumns(t *testing.T) { + parsed, err := schema.Parse(&BillingSettlement{}, &sync.Map{}, schema.NamingStrategy{}) + require.NoError(t, err) + + updates := billingSettlementReviewUpdates(7003, false, "reviewed", time.Unix(123, 0)) + for key := range updates { + assert.NotNilf(t, parsed.LookUpField(key), "review update key %q must map to a BillingSettlement column", key) + } +} + +func setBillingReconciliationBlockDefaultForTest(t *testing.T, enabled bool) { + t.Helper() + key := billing_reconciliation_setting.OptionKeyBlockUserByDefault + common.OptionMapRWMutex.Lock() + if common.OptionMap == nil { + common.OptionMap = make(map[string]string) + } + original, existed := common.OptionMap[key] + common.OptionMap[key] = map[bool]string{true: "true", false: "false"}[enabled] + common.OptionMapRWMutex.Unlock() + t.Cleanup(func() { + common.OptionMapRWMutex.Lock() + defer common.OptionMapRWMutex.Unlock() + if existed { + common.OptionMap[key] = original + } else { + delete(common.OptionMap, key) + } + }) +} + func TestBillingRequestFinalizeOperationKey(t *testing.T) { assert.Equal(t, "request:example-request:finalize", BillingRequestFinalizeOperationKey("example-request")) assert.Equal(t, "request:%:finalize", BillingRequestFinalizeOperationKey("%")) diff --git a/router/api-router.go b/router/api-router.go index 99c6efd1..1ca0a486 100644 --- a/router/api-router.go +++ b/router/api-router.go @@ -234,6 +234,9 @@ func SetApiRouter(router *gin.Engine) { smartOpsRoute.Use(middleware.AdminAuth()) { smartOpsRoute.GET("/alerts", controller.GetSmartOpsAlerts) + smartOpsRoute.GET("/billing-settlements", middleware.DisableCache(), controller.GetBillingSettlementReconciliation) + smartOpsRoute.PUT("/billing-settlements/blocking-policy", middleware.CriticalRateLimit(), middleware.UserCriticalRateLimit(), middleware.DisableCache(), anonymousRequestBodyLimit, controller.UpdateBillingSettlementBlockingPolicy) + smartOpsRoute.POST("/billing-settlements/:id/review", middleware.CriticalRateLimit(), middleware.UserCriticalRateLimit(), middleware.DisableCache(), anonymousRequestBodyLimit, controller.ReviewBillingSettlement) smartOpsRoute.GET("/channel-performance", controller.GetChannelPerformance) smartOpsRoute.GET("/channel-performance/detail", controller.GetChannelPerformanceDetail) smartOpsRoute.GET("/model-performance", controller.GetModelPerformance) diff --git a/service/billing_session.go b/service/billing_session.go index 0798c588..a0d53156 100644 --- a/service/billing_session.go +++ b/service/billing_session.go @@ -88,18 +88,20 @@ func observeBillingSettlementBacklog(stats model.BillingSettlementBacklogStats, billingSettlementBacklogAlertState.oldestCreatedAt = 0 billingSettlementBacklogAlertState.lastNotifiedAt = time.Time{} billingSettlementBacklogAlertState.Unlock() + resolvedAlert := SmartOpsAlert{ + Key: "billing_settlement_backlog", + Status: smartOpsAlertStatusResolved, + Severity: smartOpsAlertSeverityWarning, + Component: "billing", + Node: smartOpsAlertNodeName(), + CurrentValue: 0, + Threshold: 0, + ObservedAt: observedAt, + Message: fmt.Sprintf("未完成的正向最终计费对账已清空，此前共 %d 条。", previousCount), + } + projectSmartOpsActiveAlert(resolvedAlert) if wasActive { - billingSettlementBacklogAlertNotificationSender(SmartOpsAlert{ - Key: "billing_settlement_backlog", - Status: smartOpsAlertStatusResolved, - Severity: smartOpsAlertSeverityWarning, - Component: "billing", - Node: smartOpsAlertNodeName(), - CurrentValue: 0, - Threshold: 0, - ObservedAt: observedAt, - Message: fmt.Sprintf("未完成的正向最终计费对账已清空，此前共 %d 条。", previousCount), - }) + billingSettlementBacklogAlertNotificationSender(resolvedAlert) } return } @@ -123,11 +125,7 @@ func observeBillingSettlementBacklog(stats model.BillingSettlementBacklogStats, billingSettlementBacklogAlertState.lastNotifiedAt = observedAt } billingSettlementBacklogAlertState.Unlock() - if !shouldNotify { - return - } - - billingSettlementBacklogAlertNotificationSender(SmartOpsAlert{ + alert := SmartOpsAlert{ Key: "billing_settlement_backlog", Status: smartOpsAlertStatusFiring, Severity: smartOpsAlertSeverityWarning, @@ -137,11 +135,15 @@ func observeBillingSettlementBacklog(stats model.BillingSettlementBacklogStats, Threshold: oldestAge.Seconds(), ObservedAt: observedAt, Message: fmt.Sprintf( - "存在 %d 条未完成的正向最终计费对账，最早已等待 %s。相关用户的新付费请求会被阻止，请尽快处理 pending/manual 记录。", + "存在 %d 条尚未核对的正向最终计费对账，最早已等待 %s。是否阻止相关用户的新付费请求由管理员配置和单条核对策略决定，请尽快处理 pending/manual 记录。", stats.Count, oldestAge.Round(time.Second), ), - }) + } + projectSmartOpsActiveAlert(alert) + if shouldNotify { + billingSettlementBacklogAlertNotificationSender(alert) + } } func (s *BillingSession) notifyFundingOutcomeUnknown(requested, applied int64) { diff --git a/service/billing_session_test.go b/service/billing_session_test.go index d552b659..6909b8c4 100644 --- a/service/billing_session_test.go +++ b/service/billing_session_test.go @@ -41,6 +41,10 @@ type uncertainFundingSource struct { func TestBillingSettlementBacklogAlertReportsCountAgeAndRecovery(t *testing.T) { originalSender := billingSettlementBacklogAlertNotificationSender + smartOpsAlertMonitor.Lock() + originalActiveAlerts := smartOpsAlertMonitor.active + smartOpsAlertMonitor.active = make(map[string]SmartOpsAlert) + smartOpsAlertMonitor.Unlock() billingSettlementBacklogAlertState.Lock() originalState := struct { active bool @@ -60,6 +64,9 @@ func TestBillingSettlementBacklogAlertReportsCountAgeAndRecovery(t *testing.T) { billingSettlementBacklogAlertState.Unlock() t.Cleanup(func() { billingSettlementBacklogAlertNotificationSender = originalSender + smartOpsAlertMonitor.Lock() + smartOpsAlertMonitor.active = originalActiveAlerts + smartOpsAlertMonitor.Unlock() billingSettlementBacklogAlertState.Lock() billingSettlementBacklogAlertState.active = originalState.active billingSettlementBacklogAlertState.lastCount = originalState.lastCount @@ -83,6 +90,10 @@ func TestBillingSettlementBacklogAlertReportsCountAgeAndRecovery(t *testing.T) { assert.Equal(t, (2 * time.Hour).Seconds(), alerts[0].Threshold) assert.Contains(t, alerts[0].Message, "2 条") assert.Contains(t, alerts[0].Message, "2h0m0s") + activeAlerts := GetSmartOpsAlerts() + require.Len(t, activeAlerts, 1) + assert.Equal(t, "billing_settlement_backlog", activeAlerts[0].Key) + assert.Equal(t, float64(2), activeAlerts[0].CurrentValue) observeBillingSettlementBacklog(stats, now.Add(time.Minute)) require.Len(t, alerts, 1, "an unchanged backlog must be deduplicated inside the reminder interval") @@ -91,6 +102,9 @@ func TestBillingSettlementBacklogAlertReportsCountAgeAndRecovery(t *testing.T) { observeBillingSettlementBacklog(stats, now.Add(2*time.Minute)) require.Len(t, alerts, 2) assert.Equal(t, float64(3), alerts[1].CurrentValue) + activeAlerts = GetSmartOpsAlerts() + require.Len(t, activeAlerts, 1) + assert.Equal(t, float64(3), activeAlerts[0].CurrentValue) observeBillingSettlementBacklog(stats, now.Add(2*time.Minute+billingSettlementBacklogNotificationInterval)) require.Len(t, alerts, 3, "a persistent backlog must produce an age reminder") @@ -99,6 +113,7 @@ func TestBillingSettlementBacklogAlertReportsCountAgeAndRecovery(t *testing.T) { require.Len(t, alerts, 4) assert.Equal(t, smartOpsAlertStatusResolved, alerts[3].Status) assert.Contains(t, alerts[3].Message, "此前共 3 条") + assert.Empty(t, GetSmartOpsAlerts()) } func (f *uncertainFundingSource) Source() string { return BillingSourceWallet } diff --git a/service/smart_ops_alert.go b/service/smart_ops_alert.go index 3f61dc80..50df6890 100644 --- a/service/smart_ops_alert.go +++ b/service/smart_ops_alert.go @@ -2,17 +2,24 @@ package service import ( "context" + "errors" "fmt" "hash/fnv" "sort" + "strconv" + "strings" "sync" "time" "github.com/MAX-API-Next/MAX-API/common" "github.com/MAX-API-Next/MAX-API/dto" "github.com/MAX-API-Next/MAX-API/model" + "github.com/MAX-API-Next/MAX-API/setting/billing_reconciliation_setting" ) +var ErrInvalidBillingSettlementReconciliationQuery = errors.New("invalid billing settlement reconciliation query") +var ErrInvalidBillingSettlementReconciliationReview = errors.New("invalid billing settlement reconciliation review") + const ( smartOpsAlertStatusFiring = "firing" smartOpsAlertStatusResolved = "resolved" @@ -30,8 +37,8 @@ var smartOpsAlertRetryDelays = []time.Duration{ } // SmartOpsAlert is the read-only incident projection exposed to administrators. -// It intentionally contains only resource metadata and never includes secrets, -// request bodies, channel keys, or billing data. +// It intentionally excludes secrets, request bodies, channel keys, and +// mutation controls. Component-specific values retain their own units. type SmartOpsAlert struct { Key string `json:"key"` Status string `json:"status"` @@ -154,7 +161,10 @@ func StartSmartOpsAlertMonitor() { smartOpsAlertMonitor.cancel = cancel smartOpsAlertMonitor.done = done smartOpsAlertMonitor.states = make(map[string]*smartOpsAlertState) - smartOpsAlertMonitor.active = make(map[string]SmartOpsAlert) + if smartOpsAlertMonitor.active == nil { + smartOpsAlertMonitor.active = make(map[string]SmartOpsAlert) + } + clearSmartOpsSystemAlertsLocked() go func() { defer close(done) runSmartOpsAlertMonitor(ctx) @@ -206,7 +216,7 @@ func evaluateSmartOpsSystemAlerts(status common.SystemStatus, config common.Perf for _, state := range smartOpsAlertMonitor.states { state.suppress(status.ObservedAt) } - clear(smartOpsAlertMonitor.active) + clearSmartOpsSystemAlertsLocked() smartOpsAlertMonitor.Unlock() return } @@ -282,6 +292,25 @@ func evaluateSmartOpsSystemAlerts(status common.SystemStatus, config common.Perf } } +func clearSmartOpsSystemAlertsLocked() { + for _, definition := range smartOpsAlertDefinitions { + delete(smartOpsAlertMonitor.active, definition.key) + } +} + +func projectSmartOpsActiveAlert(alert SmartOpsAlert) { + smartOpsAlertMonitor.Lock() + defer smartOpsAlertMonitor.Unlock() + if smartOpsAlertMonitor.active == nil { + smartOpsAlertMonitor.active = make(map[string]SmartOpsAlert) + } + if alert.Status == smartOpsAlertStatusResolved { + delete(smartOpsAlertMonitor.active, alert.Key) + return + } + smartOpsAlertMonitor.active[alert.Key] = alert +} + func formatSmartOpsAlertMessage(label string, event *smartOpsAlertEvent) string { if event.Status == smartOpsAlertStatusResolved { return fmt.Sprintf("%s 已恢复：当前 %.1f%%，阈值 %.1f%%", label, event.CurrentValue, event.Threshold) @@ -311,6 +340,49 @@ func GetSmartOpsAlerts() []SmartOpsAlert { return alerts } +// GetBillingSettlementReconciliation returns bounded, read-only settlement +// evidence for administrators. It never retries or mutates financial state. +func GetBillingSettlementReconciliation(limit int) (model.BillingSettlementReconciliationData, error) { + if limit == 0 { + limit = 100 + } + if limit < 1 || limit > 200 { + return model.BillingSettlementReconciliationData{}, fmt.Errorf( + "%w: limit must be between 1 and 200", + ErrInvalidBillingSettlementReconciliationQuery, + ) + } + return model.GetUnresolvedPositiveFinalizeSettlements(limit) +} + +func UpdateBillingSettlementBlockingPolicy(blockUserByDefault bool) error { + return model.UpdateOption( + billing_reconciliation_setting.OptionKeyBlockUserByDefault, + strconv.FormatBool(blockUserByDefault), + ) +} + +func ReviewBillingSettlement(id int64, reviewerID int, blockUser *bool, note string) (model.BillingSettlement, error) { + if id <= 0 || reviewerID <= 0 || blockUser == nil { + return model.BillingSettlement{}, ErrInvalidBillingSettlementReconciliationReview + } + note = strings.TrimSpace(note) + noteLength := len([]rune(note)) + if noteLength < 3 || noteLength > 1000 { + return model.BillingSettlement{}, fmt.Errorf( + "%w: note must contain between 3 and 1000 characters", + ErrInvalidBillingSettlementReconciliationReview, + ) + } + note = common.SanitizePersistedLogContent(common.MaskSensitiveInfo(note)) + record, err := model.ReviewBillingSettlement(id, reviewerID, *blockUser, note) + if err != nil { + return model.BillingSettlement{}, err + } + model.ObserveBillingSettlementBacklogNow() + return record, nil +} + type smartOpsAlertRecipient struct { ID int Email string diff --git a/service/smart_ops_alert_test.go b/service/smart_ops_alert_test.go index 3e2ba9a5..0629597d 100644 --- a/service/smart_ops_alert_test.go +++ b/service/smart_ops_alert_test.go @@ -123,6 +123,47 @@ func TestEvaluateSmartOpsSystemAlertsDoesNotResolveFromInvalidProbe(t *testing.T require.Len(t, GetSmartOpsAlerts(), 1, "an unknown sample must preserve the active incident") } +func TestDisablingSystemMonitoringPreservesBillingAlerts(t *testing.T) { + smartOpsAlertMonitor.Lock() + originalStates := smartOpsAlertMonitor.states + originalActive := smartOpsAlertMonitor.active + smartOpsAlertMonitor.states = make(map[string]*smartOpsAlertState) + smartOpsAlertMonitor.active = map[string]SmartOpsAlert{ + "billing_settlement_backlog": { + Key: "billing_settlement_backlog", + Status: smartOpsAlertStatusFiring, + Component: "billing", + }, + } + smartOpsAlertMonitor.Unlock() + t.Cleanup(func() { + smartOpsAlertMonitor.Lock() + smartOpsAlertMonitor.states = originalStates + smartOpsAlertMonitor.active = originalActive + smartOpsAlertMonitor.Unlock() + }) + + evaluateSmartOpsSystemAlerts(common.SystemStatus{}, common.PerformanceMonitorConfig{Enabled: false}, time.Now()) + + alerts := GetSmartOpsAlerts() + require.Len(t, alerts, 1) + require.Equal(t, "billing_settlement_backlog", alerts[0].Key) +} + +func TestBillingSettlementReconciliationRejectsUnboundedLimit(t *testing.T) { + _, err := GetBillingSettlementReconciliation(201) + require.ErrorIs(t, err, ErrInvalidBillingSettlementReconciliationQuery) +} + +func TestBillingSettlementReviewRequiresExplicitPolicyAndReason(t *testing.T) { + _, err := ReviewBillingSettlement(1, 2, nil, "verified") + require.ErrorIs(t, err, ErrInvalidBillingSettlementReconciliationReview) + + block := false + _, err = ReviewBillingSettlement(1, 2, &block, "x") + require.ErrorIs(t, err, ErrInvalidBillingSettlementReconciliationReview) +} + func TestSmartOpsAlertNotifyTypeSeparatesNodes(t *testing.T) { alert := SmartOpsAlert{Key: "system_cpu", Node: "node-a", Status: smartOpsAlertStatusFiring} nodeAType := smartOpsAlertNotifyType(alert) diff --git a/setting/billing_reconciliation_setting/config.go b/setting/billing_reconciliation_setting/config.go new file mode 100644 index 00000000..b2d79d50 --- /dev/null +++ b/setting/billing_reconciliation_setting/config.go @@ -0,0 +1,39 @@ +package billing_reconciliation_setting + +import ( + "strconv" + + "github.com/MAX-API-Next/MAX-API/common" + "github.com/MAX-API-Next/MAX-API/setting/config" +) + +const OptionKeyBlockUserByDefault = "billing_reconciliation_setting.block_user_by_default" + +type BillingReconciliationSetting struct { + BlockUserByDefault bool `json:"block_user_by_default"` +} + +var billingReconciliationSetting = BillingReconciliationSetting{ + BlockUserByDefault: true, +} + +func init() { + config.GlobalConfig.Register("billing_reconciliation_setting", &billingReconciliationSetting) +} + +// BlockUserByDefault returns the persisted admission policy. Reading through +// OptionMap keeps the hot path synchronized with runtime option updates while +// retaining a fail-closed default before options are initialized. +func BlockUserByDefault() bool { + common.OptionMapRWMutex.RLock() + raw, ok := common.OptionMap[OptionKeyBlockUserByDefault] + common.OptionMapRWMutex.RUnlock() + if !ok { + return true + } + enabled, err := strconv.ParseBool(raw) + if err != nil { + return true + } + return enabled +} diff --git a/setting/billing_reconciliation_setting/config_test.go b/setting/billing_reconciliation_setting/config_test.go new file mode 100644 index 00000000..8b4e1527 --- /dev/null +++ b/setting/billing_reconciliation_setting/config_test.go @@ -0,0 +1,15 @@ +package billing_reconciliation_setting + +import ( + "testing" + + "github.com/MAX-API-Next/MAX-API/setting/config" + "github.com/stretchr/testify/assert" +) + +func TestDefaultBlockingPolicyIsRegisteredFailClosed(t *testing.T) { + registered := config.GlobalConfig.ExportAllConfigs() + + assert.Equal(t, "true", registered[OptionKeyBlockUserByDefault]) + assert.True(t, BlockUserByDefault()) +} diff --git a/web/default/src/features/smart-ops/alerts.test.tsx b/web/default/src/features/smart-ops/alerts.test.tsx index d85570b8..f449c040 100644 --- a/web/default/src/features/smart-ops/alerts.test.tsx +++ b/web/default/src/features/smart-ops/alerts.test.tsx @@ -18,7 +18,7 @@ For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API */ import { QueryClient, QueryClientProvider } from '@tanstack/react-query' import { createReactTestEnvironment } from '@/test/react' -import { waitFor, within } from '@testing-library/react' +import { fireEvent, waitFor, within } from '@testing-library/react' import assert from 'node:assert/strict' import { after, before, describe, test } from 'node:test' import { api } from '@/lib/api' @@ -49,12 +49,32 @@ function createQueryClient(): QueryClient { }) } +function emptyReconciliationData() { + return { + total_count: 0, + pending_count: 0, + manual_count: 0, + open_alert_count: 0, + reviewed_count: 0, + blocking_record_count: 0, + blocked_user_count: 0, + block_user_by_default: true, + oldest_created_at: 0, + truncated: false, + generated_at: 1788106455, + items: [], + } +} + describe('SmartOps active alerts', () => { test('polls the administrator alert endpoint and renders active host pressure', async () => { const originalGet = api.get const urls: string[] = [] api.get = (async (url) => { urls.push(String(url)) + if (url === '/api/smart-ops/billing-settlements') { + return { data: { success: true, data: emptyReconciliationData() } } + } return { data: { success: true, @@ -102,10 +122,13 @@ describe('SmartOps active alerts', () => { test('formats percentages with the active locale', async () => { const originalGet = api.get await testEnv.i18n.changeLanguage('fr') - api.get = (async () => ({ - data: { - success: true, - data: [ + api.get = (async (url) => ({ + data: + url === '/api/smart-ops/billing-settlements' + ? { success: true, data: emptyReconciliationData() } + : { + success: true, + data: [ { key: 'system_memory', status: 'firing', @@ -117,8 +140,8 @@ describe('SmartOps active alerts', () => { observed_at: '2026-08-22T08:00:00Z', message: 'Memory usage exceeded the threshold', }, - ], - }, + ], + }, })) as typeof api.get const queryClient = createQueryClient() @@ -140,10 +163,296 @@ describe('SmartOps active alerts', () => { } }) + test('renders billing backlog values and read-only reconciliation evidence', async () => { + const originalGet = api.get + const urls: string[] = [] + api.get = (async (url) => { + urls.push(String(url)) + if (url === '/api/smart-ops/billing-settlements') { + return { + data: { + success: true, + data: { + total_count: 48, + pending_count: 17, + manual_count: 31, + open_alert_count: 1, + reviewed_count: 47, + blocking_record_count: 1, + blocked_user_count: 9, + block_user_by_default: true, + oldest_created_at: 1786032544, + truncated: false, + generated_at: 1788106455, + items: [ + { + id: 71, + operation_key: 'request:billing-request-71:finalize', + status: 'manual', + source: 'wallet', + user_id: 42, + subscription_id: 0, + token_id: 84, + task_id: 0, + funding_delta: 2500, + applied_funding_delta: 0, + token_delta: 2500, + applied_token_delta: 0, + attempts: 1, + last_error: 'user quota is not enough', + next_attempt: 0, + created_at: 1786032544, + updated_at: 1786032544, + reconciliation_reviewed_at: 0, + reconciliation_reviewed_by: 0, + reconciliation_review_note: '', + user_blocking_override: null, + blocks_user: true, + }, + ], + }, + }, + } + } + return { + data: { + success: true, + data: [ + { + key: 'billing_settlement_backlog', + status: 'firing', + severity: 'warning', + component: 'billing', + node: 'XG', + current_value: 48, + threshold: 2121911, + observed_at: '2026-08-30T16:14:15Z', + message: 'billing backlog', + }, + ], + }, + } + }) as typeof api.get + + const queryClient = createQueryClient() + const view = await testEnv.render( + + + + ) + + try { + await waitFor(() => { + const text = view.container.textContent ?? '' + assert.ok(urls.includes('/api/smart-ops/alerts')) + assert.ok(urls.includes('/api/smart-ops/billing-settlements')) + assert.ok(text.includes('Billing reconciliation backlog')) + assert.ok(text.includes('48 records')) + assert.ok(!text.includes('4,800.0%')) + assert.ok(text.includes('Pending: 17')) + assert.ok(text.includes('Manual: 31')) + assert.ok(text.includes('Blocked users: 9')) + assert.ok(text.includes('request:billing-request-71:finalize')) + assert.ok(text.includes('user quota is not enough')) + }) + } finally { + api.get = originalGet + queryClient.clear() + await view.unmount() + } + }) + + test('keeps the billing alert visible when reconciliation details fail', async () => { + const originalGet = api.get + api.get = (async (url) => { + if (url === '/api/smart-ops/billing-settlements') { + throw new Error('temporary reconciliation projection failure') + } + return { + data: { + success: true, + data: [ + { + key: 'billing_settlement_backlog', + status: 'firing', + severity: 'warning', + component: 'billing', + node: 'XG', + current_value: 48, + threshold: 2121911, + observed_at: '2026-08-30T16:14:15Z', + message: 'billing backlog', + }, + ], + }, + } + }) as typeof api.get + + const queryClient = createQueryClient() + const view = await testEnv.render( + + + + ) + + try { + await waitFor(() => { + const text = view.container.textContent ?? '' + assert.ok(text.includes('Billing reconciliation backlog')) + assert.ok(text.includes('48 records')) + assert.ok( + text.includes('We could not load billing reconciliation details.') + ) + assert.ok(text.includes('temporary reconciliation projection failure')) + }) + } finally { + api.get = originalGet + queryClient.clear() + await view.unmount() + } + }) + + test('loads reconciliation without an active alert and submits administrator controls', async () => { + const originalGet = api.get + const originalPut = api.put + const originalPost = api.post + const writes: Array<{ method: string; url: string; data: unknown }> = [] + let reconciliationRequests = 0 + api.get = (async (url) => { + if (url === '/api/smart-ops/billing-settlements') { + reconciliationRequests += 1 + return { + data: { + success: true, + data: { + ...emptyReconciliationData(), + total_count: 1, + pending_count: 1, + open_alert_count: 1, + blocking_record_count: 1, + blocked_user_count: 1, + items: [ + { + id: 91, + operation_key: 'request:billing-request-91:finalize', + status: 'pending', + source: 'wallet', + user_id: 51, + subscription_id: 0, + token_id: 0, + task_id: 0, + funding_delta: 100, + applied_funding_delta: 0, + token_delta: 100, + applied_token_delta: 0, + attempts: 2, + last_error: 'quota changed', + next_attempt: 1788106500, + created_at: 1786032544, + updated_at: 1786032544, + reconciliation_reviewed_at: 0, + reconciliation_reviewed_by: 0, + reconciliation_review_note: '', + user_blocking_override: null, + blocks_user: true, + }, + ], + }, + }, + } + } + return { data: { success: true, data: [] } } + }) as typeof api.get + api.put = (async (url, data) => { + writes.push({ method: 'PUT', url: String(url), data }) + return { data: { success: true } } + }) as typeof api.put + api.post = (async (url, data) => { + writes.push({ method: 'POST', url: String(url), data }) + return { data: { success: true } } + }) as typeof api.post + + const queryClient = createQueryClient() + const view = await testEnv.render( + + + + ) + + try { + let policySwitch: HTMLElement | undefined + let reviewButton: HTMLElement | undefined + await waitFor(() => { + const screen = within(view.container) + assert.ok((view.container.textContent ?? '').includes('No active alerts.')) + assert.ok( + (view.container.textContent ?? '').includes( + 'request:billing-request-91:finalize' + ) + ) + policySwitch = screen.getByRole('switch', { + name: 'Block affected users by default', + }) + reviewButton = screen.getByRole('button', { + name: 'Review and close', + }) + }) + + assert.ok(policySwitch) + await view.click(policySwitch) + await waitFor(() => { + assert.deepEqual(writes[0], { + method: 'PUT', + url: '/api/smart-ops/billing-settlements/blocking-policy', + data: { block_user_by_default: false }, + }) + assert.ok(reconciliationRequests > 1) + }) + + reviewButton = within(view.container).getByRole('button', { + name: 'Review and close', + }) + fireEvent.click(reviewButton) + const dialog = within(document.body).getByRole('dialog') + const allowButton = within(dialog).getByRole('button', { + name: 'Allow user to continue', + }) + await view.click(allowButton) + fireEvent.change( + within(dialog).getByRole('textbox', { name: 'Review note' }), + { target: { value: 'Matched provider invoice and usage logs' } } + ) + await view.click( + within(dialog).getByRole('button', { name: 'Close alert' }) + ) + + await waitFor(() => { + assert.deepEqual(writes[1], { + method: 'POST', + url: '/api/smart-ops/billing-settlements/91/review', + data: { + block_user: false, + note: 'Matched provider invoice and usage logs', + }, + }) + assert.ok(reconciliationRequests > 1) + }) + } finally { + api.get = originalGet + api.put = originalPut + api.post = originalPost + queryClient.clear() + await view.unmount() + } + }) + test('shows a healthy empty state when no incident is active', async () => { const originalGet = api.get - api.get = (async () => ({ - data: { success: true, data: [] }, + api.get = (async (url) => ({ + data: + url === '/api/smart-ops/billing-settlements' + ? { success: true, data: emptyReconciliationData() } + : { success: true, data: [] }, })) as typeof api.get const queryClient = createQueryClient() @@ -169,8 +478,11 @@ describe('SmartOps active alerts', () => { test('shows a localized fallback for an unsuccessful response', async () => { const originalGet = api.get await testEnv.i18n.changeLanguage('fr') - api.get = (async () => ({ - data: { success: false, data: [] }, + api.get = (async (url) => ({ + data: + url === '/api/smart-ops/billing-settlements' + ? { success: true, data: emptyReconciliationData() } + : { success: false, data: [] }, })) as typeof api.get const queryClient = createQueryClient() @@ -197,7 +509,10 @@ describe('SmartOps active alerts', () => { test('shows a retryable error state when the alert endpoint fails', async () => { const originalGet = api.get let requestCount = 0 - api.get = (async () => { + api.get = (async (url) => { + if (url === '/api/smart-ops/billing-settlements') { + return { data: { success: true, data: emptyReconciliationData() } } + } requestCount += 1 throw new Error('temporary alert endpoint failure') }) as typeof api.get diff --git a/web/default/src/features/smart-ops/alerts.tsx b/web/default/src/features/smart-ops/alerts.tsx index 9f53e8b0..134c3805 100644 --- a/web/default/src/features/smart-ops/alerts.tsx +++ b/web/default/src/features/smart-ops/alerts.tsx @@ -16,14 +16,38 @@ along with this program. If not, see . For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues */ -import type { ReactElement } from 'react' +import { useEffect, useState, type ReactElement } from 'react' import { useQuery } from '@tanstack/react-query' -import { RefreshCw, ShieldCheck, TriangleAlert } from 'lucide-react' +import type { TFunction } from 'i18next' +import { + CheckCircle2, + Loader2, + LockKeyhole, + LockOpen, + Pencil, + RefreshCw, + ShieldCheck, + TriangleAlert, +} from 'lucide-react' import { useTranslation } from 'react-i18next' -import { formatTimestampRelative, formatTimestampToDate } from '@/lib/format' +import { toast } from 'sonner' +import { + formatQuota, + formatTimestampRelative, + formatTimestampToDate, +} from '@/lib/format' import { cn } from '@/lib/utils' +import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert' import { Badge } from '@/components/ui/badge' import { Button } from '@/components/ui/button' +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from '@/components/ui/dialog' import { Empty, EmptyDescription, @@ -31,7 +55,17 @@ import { EmptyMedia, EmptyTitle, } from '@/components/ui/empty' +import { + Field, + FieldContent, + FieldDescription, + FieldError, + FieldGroup, + FieldLabel, + FieldTitle, +} from '@/components/ui/field' import { Skeleton } from '@/components/ui/skeleton' +import { Switch } from '@/components/ui/switch' import { Table, TableBody, @@ -40,17 +74,38 @@ import { TableHeader, TableRow, } from '@/components/ui/table' +import { Textarea } from '@/components/ui/textarea' +import { + ToggleGroup, + ToggleGroupItem, +} from '@/components/ui/toggle-group' import { ErrorState } from '@/components/error-state' import { SectionPageLayout } from '@/components/layout' -import { getSmartOpsAlerts } from './api' -import type { SmartOpsAlert } from './types' +import { + getBillingSettlementReconciliation, + getSmartOpsAlerts, + reviewBillingSettlement, + updateBillingSettlementBlockingPolicy, +} from './api' +import type { + BillingSettlementReconciliationData, + BillingSettlementReconciliationItem, + SmartOpsAlert, +} from './types' -const ALERT_POLL_INTERVAL_MS = 5000 +const ALERT_POLL_INTERVAL_MS = 30000 +const BILLING_BACKLOG_ALERT_KEY = 'billing_settlement_backlog' const RESOURCE_LABEL: Record = { system_cpu: 'CPU usage', system_memory: 'Memory usage', system_disk: 'Disk usage', + [BILLING_BACKLOG_ALERT_KEY]: 'Billing reconciliation backlog', +} + +const FUNDING_SOURCE_LABEL: Record = { + wallet: 'Wallet', + subscription: 'Subscription', } function formatPercent(value: number, locale: string): string { @@ -60,11 +115,76 @@ function formatPercent(value: number, locale: string): string { }).format(value)}%` } +function formatCount(value: number, locale: string): string { + return new Intl.NumberFormat(locale, { maximumFractionDigits: 0 }).format( + value + ) +} + +function formatDurationSeconds( + value: number, + locale: string, + t: TFunction +): string { + const totalSeconds = Math.max(0, Math.floor(value)) + const days = Math.floor(totalSeconds / 86400) + const hours = Math.floor((totalSeconds % 86400) / 3600) + const minutes = Math.floor((totalSeconds % 3600) / 60) + const seconds = totalSeconds % 60 + const units: Array<[number, string]> = [ + [days, '{{count}} days'], + [hours, '{{count}} hours'], + [minutes, '{{count}} minutes'], + [seconds, '{{count}} seconds'], + ] + const parts = units + .filter(([count]) => count > 0) + .slice(0, 2) + .map(([count, key]) => + t(key, { + count: formatCount(count, locale), + interpolation: { escapeValue: false }, + }) + ) + return parts.length > 0 + ? parts.join(' ') + : t('{{count}} seconds', { count: 0 }) +} + function getObservedAtMilliseconds(alert: SmartOpsAlert): number | undefined { const timestamp = Date.parse(alert.observed_at) return Number.isNaN(timestamp) ? undefined : timestamp } +function isBillingBacklogAlert(alert: SmartOpsAlert): boolean { + return alert.key === BILLING_BACKLOG_ALERT_KEY +} + +function formatAlertCurrentValue( + alert: SmartOpsAlert, + locale: string, + t: TFunction +): string { + if (isBillingBacklogAlert(alert)) { + return t('{{count}} records', { + count: formatCount(alert.current_value, locale), + interpolation: { escapeValue: false }, + }) + } + return formatPercent(alert.current_value, locale) +} + +function formatAlertThreshold( + alert: SmartOpsAlert, + locale: string, + t: TFunction +): string { + if (isBillingBacklogAlert(alert)) { + return formatDurationSeconds(alert.threshold, locale, t) + } + return formatPercent(alert.threshold, locale) +} + interface ActiveAlertsTableProps { alerts: SmartOpsAlert[] } @@ -74,7 +194,7 @@ function ActiveAlertsTable(props: ActiveAlertsTableProps): ReactElement { return (
- +
{t('Resource')} @@ -84,7 +204,7 @@ function ActiveAlertsTable(props: ActiveAlertsTableProps): ReactElement { {t('Current Value')} - {t('Threshold')} + {t('Threshold / Oldest wait')} {t('Observed')} @@ -96,8 +216,8 @@ function ActiveAlertsTable(props: ActiveAlertsTableProps): ReactElement { const observedAt = getObservedAtMilliseconds(alert) return ( - -
+ +
{t(RESOURCE_LABEL[alert.key] ?? alert.key)} @@ -113,10 +233,10 @@ function ActiveAlertsTable(props: ActiveAlertsTableProps): ReactElement { {alert.node || t('Unknown')} - {formatPercent(alert.current_value, i18n.language)} + {formatAlertCurrentValue(alert, i18n.language, t)} - {formatPercent(alert.threshold, i18n.language)} + {formatAlertThreshold(alert, i18n.language, t)} 0) { + references.push(t('Subscription #{{id}}', { id: item.subscription_id })) + } + if (item.token_id > 0) { + references.push(t('Token #{{id}}', { id: item.token_id })) + } + if (item.task_id > 0) { + references.push(t('Task #{{id}}', { id: item.task_id })) + } + return references +} + +interface BillingSettlementEvidenceProps { + data?: BillingSettlementReconciliationData + error: Error | null + loading: boolean + onRetry: () => void + onChanged: () => Promise +} + +interface SettlementReviewDialogProps { + item: BillingSettlementReconciliationItem | null + open: boolean + onOpenChange: (open: boolean) => void + onReviewed: () => Promise +} + +function mutationErrorMessage(error: unknown, fallback: string): string { + if ( + typeof error === 'object' && + error !== null && + 'response' in error && + typeof error.response === 'object' && + error.response !== null && + 'data' in error.response && + typeof error.response.data === 'object' && + error.response.data !== null && + 'message' in error.response.data && + typeof error.response.data.message === 'string' + ) { + return error.response.data.message + } + return error instanceof Error ? error.message : fallback +} + +function SettlementReviewDialog( + props: SettlementReviewDialogProps +): ReactElement { + const { t } = useTranslation() + const [decision, setDecision] = useState<'block' | 'allow'>('block') + const [note, setNote] = useState('') + const [saving, setSaving] = useState(false) + + useEffect(() => { + if (!props.open || !props.item) return + setDecision(props.item.blocks_user ? 'block' : 'allow') + setNote(props.item.reconciliation_review_note || '') + }, [props.item, props.open]) + + const noteLength = note.trim().length + const noteInvalid = noteLength > 0 && (noteLength < 3 || noteLength > 1000) + + const handleSubmit = async () => { + if (!props.item || noteLength < 3 || noteLength > 1000) return + setSaving(true) + try { + const response = await reviewBillingSettlement(props.item.id, { + block_user: decision === 'block', + note: note.trim(), + }) + if (!response.success) { + throw new Error(response.message || t('Failed to save review.')) + } + toast.success(t('Billing reconciliation alert closed.')) + props.onOpenChange(false) + await props.onReviewed() + } catch (error) { + toast.error(mutationErrorMessage(error, t('Failed to save review.'))) + } finally { + setSaving(false) + } + } + + return ( + + + + + {props.item?.reconciliation_reviewed_at + ? t('Edit reconciliation review') + : t('Review billing reconciliation')} + + + {t( + 'Closing this alert records an administrator review only. It does not mark the settlement as applied or change any balance.' + )} + + + + + + {t('User access while unresolved')} + { + const nextDecision = value[0] + if (nextDecision === 'block' || nextDecision === 'allow') { + setDecision(nextDecision) + } + }} + variant='outline' + spacing={2} + className='grid w-full grid-cols-1 sm:grid-cols-2' + > + + + + + + + {t( + 'This choice affects only new paid-request admission while the financial settlement remains pending or manual.' + )} + + + + + + {t('Review note')} + +