diff --git a/controller/audit.go b/controller/audit.go index fac35107..bf488d84 100644 --- a/controller/audit.go +++ b/controller/audit.go @@ -31,6 +31,8 @@ var auditContentTemplates = map[string]string{ "user.topup_complete": "Completed top-up order ${tradeNo}", "option.update": "Updated system setting ${key}", "option.reset_ratio": "Reset model ratio settings", + "billing.reconciliation_blocking_policy_update": "Updated billing reconciliation default user-blocking policy to ${block_user_by_default}", + "billing.reconciliation_review": "Reviewed billing settlement ${settlement_id} (block user: ${block_user})", "channel.create": "Created channel ${name} (type ${type}, count ${count})", "channel.update": "Updated channel ${name} (ID: ${id})", diff --git a/controller/smart_ops_alert.go b/controller/smart_ops_alert.go index db14a395..08bfdee5 100644 --- a/controller/smart_ops_alert.go +++ b/controller/smart_ops_alert.go @@ -1,8 +1,13 @@ package controller import ( + "errors" + "fmt" "net/http" + "strconv" + "github.com/MAX-API-Next/MAX-API/common" + "github.com/MAX-API-Next/MAX-API/model" "github.com/MAX-API-Next/MAX-API/service" "github.com/gin-gonic/gin" ) @@ -15,3 +20,120 @@ func GetSmartOpsAlerts(c *gin.Context) { "data": service.GetSmartOpsAlerts(), }) } + +type billingSettlementBlockingPolicyRequest struct { + BlockUserByDefault *bool `json:"block_user_by_default"` +} + +func UpdateBillingSettlementBlockingPolicy(c *gin.Context) { + var request billingSettlementBlockingPolicyRequest + if err := common.DecodeJson(c.Request.Body, &request); err != nil || request.BlockUserByDefault == nil { + c.JSON(http.StatusBadRequest, gin.H{ + "success": false, + "message": "block_user_by_default is required", + }) + return + } + if err := service.UpdateBillingSettlementBlockingPolicy(*request.BlockUserByDefault); err != nil { + common.SysError(fmt.Sprintf("failed to update billing reconciliation blocking policy: %v", err)) + c.JSON(http.StatusInternalServerError, gin.H{ + "success": false, + "message": "failed to update billing reconciliation blocking policy", + }) + return + } + recordManageAudit(c, "billing.reconciliation_blocking_policy_update", map[string]interface{}{ + "block_user_by_default": *request.BlockUserByDefault, + }) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": gin.H{ + "block_user_by_default": *request.BlockUserByDefault, + }, + }) +} + +type billingSettlementReviewRequest struct { + BlockUser *bool `json:"block_user"` + Note string `json:"note"` +} + +func ReviewBillingSettlement(c *gin.Context) { + id, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil || id <= 0 { + c.JSON(http.StatusBadRequest, gin.H{ + "success": false, + "message": "invalid billing settlement id", + }) + return + } + var request billingSettlementReviewRequest + if err := common.DecodeJson(c.Request.Body, &request); err != nil { + c.JSON(http.StatusBadRequest, gin.H{ + "success": false, + "message": "invalid billing settlement review request", + }) + return + } + record, err := service.ReviewBillingSettlement(id, c.GetInt("id"), request.BlockUser, request.Note) + if err != nil { + status := http.StatusInternalServerError + message := "failed to review billing settlement" + switch { + case errors.Is(err, service.ErrInvalidBillingSettlementReconciliationReview): + status = http.StatusBadRequest + message = err.Error() + case errors.Is(err, model.ErrBillingSettlementReviewConflict): + status = http.StatusConflict + message = "billing settlement is no longer pending manual reconciliation" + } + if status == http.StatusInternalServerError { + common.SysError(fmt.Sprintf("failed to review billing settlement %d: %v", id, err)) + } + c.JSON(status, gin.H{"success": false, "message": message}) + return + } + recordManageAudit(c, "billing.reconciliation_review", map[string]interface{}{ + "settlement_id": record.ID, + "target_user_id": record.UserID, + "block_user": *request.BlockUser, + }) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": gin.H{ + "id": record.ID, + "reconciliation_reviewed_at": record.ReconciliationReviewedAt, + "block_user": *request.BlockUser, + }, + }) +} + +// GetBillingSettlementReconciliation exposes bounded, read-only evidence for +// unresolved positive final settlements. It does not retry or mutate records. +func GetBillingSettlementReconciliation(c *gin.Context) { + limit, err := parseIntQuery(c, "limit") + if err != nil { + respondInvalidPerformanceQuery( + c, + err, + service.ErrInvalidBillingSettlementReconciliationQuery, + "failed to query billing settlement reconciliation", + ) + return + } + result, err := service.GetBillingSettlementReconciliation(limit) + if err != nil { + respondPerformanceError( + c, + err, + service.ErrInvalidBillingSettlementReconciliationQuery, + "failed to query billing settlement reconciliation", + ) + return + } + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": result, + }) +} diff --git a/controller/smart_ops_alert_test.go b/controller/smart_ops_alert_test.go new file mode 100644 index 00000000..8e368e18 --- /dev/null +++ b/controller/smart_ops_alert_test.go @@ -0,0 +1,116 @@ +package controller + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/MAX-API-Next/MAX-API/common" + "github.com/MAX-API-Next/MAX-API/model" + "github.com/gin-gonic/gin" + "github.com/glebarez/sqlite" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +func TestBillingSettlementMutationRequestsPreserveExplicitFalse(t *testing.T) { + var policy billingSettlementBlockingPolicyRequest + require.NoError(t, common.Unmarshal([]byte(`{"block_user_by_default":false}`), &policy)) + require.NotNil(t, policy.BlockUserByDefault) + assert.False(t, *policy.BlockUserByDefault) + + var review billingSettlementReviewRequest + require.NoError(t, common.Unmarshal([]byte(`{"block_user":false,"note":"verified"}`), &review)) + require.NotNil(t, review.BlockUser) + assert.False(t, *review.BlockUser) +} + +func TestReviewBillingSettlementAuditRecordsActingAdministrator(t *testing.T) { + gin.SetMode(gin.TestMode) + oldDB := model.DB + oldLogDB := model.LOG_DB + oldRedisEnabled := common.RedisEnabled + oldMemoryCacheEnabled := common.MemoryCacheEnabled + common.RedisEnabled = false + common.MemoryCacheEnabled = false + + dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.ReplaceAll(t.Name(), "/", "_")) + db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&model.User{}, &model.BillingSettlement{}, &model.Log{})) + model.DB = db + model.LOG_DB = db + t.Cleanup(func() { + if sqlDB, dbErr := db.DB(); dbErr == nil { + _ = sqlDB.Close() + } + model.DB = oldDB + model.LOG_DB = oldLogDB + common.RedisEnabled = oldRedisEnabled + common.MemoryCacheEnabled = oldMemoryCacheEnabled + }) + + administrator := model.User{ + Id: 7101, + Username: "billing-review-admin", + AffCode: "billing-review-admin-aff", + Role: common.RoleRootUser, + Status: common.UserStatusEnabled, + Group: "default", + } + owner := model.User{ + Id: 7102, + Username: "billing-review-owner", + AffCode: "billing-review-owner-aff", + Role: common.RoleCommonUser, + Status: common.UserStatusEnabled, + Group: "default", + } + require.NoError(t, db.Create(&administrator).Error) + require.NoError(t, db.Create(&owner).Error) + settlement := model.BillingSettlement{ + OperationKey: "request:controller-review-audit:finalize", + Source: model.BillingSettlementSourceWallet, + UserID: owner.Id, + FundingDelta: 10, + TokenDelta: 10, + Status: model.BillingSettlementStatusPending, + CreatedAt: 1, + UpdatedAt: 1, + Revision: 1, + } + require.NoError(t, db.Create(&settlement).Error) + + recorder := httptest.NewRecorder() + ctx, _ := gin.CreateTestContext(recorder) + ctx.Request = httptest.NewRequest( + http.MethodPost, + fmt.Sprintf("/api/smart-ops/billing-settlements/%d/review", settlement.ID), + bytes.NewBufferString(`{"block_user":false,"note":"Verified provider evidence"}`), + ) + ctx.Params = gin.Params{{Key: "id", Value: fmt.Sprintf("%d", settlement.ID)}} + ctx.Set("id", administrator.Id) + ctx.Set("username", administrator.Username) + ctx.Set("role", administrator.Role) + + ReviewBillingSettlement(ctx) + + require.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String()) + var audit model.Log + require.NoError(t, db.Where("type = ?", model.LogTypeManage).Order("id DESC").First(&audit).Error) + assert.Equal(t, administrator.Id, audit.UserId) + assert.Equal(t, administrator.Username, audit.Username) + var other struct { + Op struct { + Action string `json:"action"` + Params map[string]interface{} `json:"params"` + } `json:"op"` + } + require.NoError(t, common.UnmarshalJsonStr(audit.Other, &other)) + assert.Equal(t, "billing.reconciliation_review", other.Op.Action) + assert.EqualValues(t, owner.Id, other.Op.Params["target_user_id"]) +} diff --git a/model/billing_settlement.go b/model/billing_settlement.go index 51907c07..93de9fb8 100644 --- a/model/billing_settlement.go +++ b/model/billing_settlement.go @@ -2,12 +2,14 @@ package model import ( "context" + "database/sql" "errors" "fmt" "sync" "time" "github.com/MAX-API-Next/MAX-API/common" + "github.com/MAX-API-Next/MAX-API/setting/billing_reconciliation_setting" "github.com/bytedance/gopkg/util/gopool" "gorm.io/gorm" "gorm.io/gorm/clause" @@ -34,6 +36,7 @@ var ( ErrBillingSettlementTaskConflict = errors.New("billing settlement task quota conflict") ErrBillingSettlementOperationConflict = errors.New("billing settlement operation conflict") ErrBillingSettlementRecordNotDurable = errors.New("billing settlement record was not durably created") + ErrBillingSettlementReviewConflict = errors.New("billing settlement is no longer reviewable") ErrSubscriptionSettlementUnbound = errors.New("subscription settlement is not bound to its pre-consume request") ErrSubscriptionSettlementPeriodChanged = errors.New("subscription settlement crossed a quota reset period") ) @@ -70,14 +73,14 @@ type BillingSettlement struct { UserID int `gorm:"not null;default:0;index:idx_billing_settlement_admission,priority:1"` SubscriptionID int `gorm:"not null;default:0"` TokenID int `gorm:"not null;default:0"` - FundingDelta int64 `gorm:"not null;default:0;index:idx_billing_settlement_admission,priority:3;index:idx_billing_settlement_status_funding,priority:2"` + FundingDelta int64 `gorm:"not null;default:0;index:idx_billing_settlement_admission,priority:3;index:idx_billing_settlement_status_funding,priority:2;index:idx_billing_settlement_reconciliation,priority:2"` AppliedFundingDelta int64 `gorm:"not null;default:0"` TokenDelta int64 `gorm:"not null;default:0"` AppliedTokenDelta int64 `gorm:"not null;default:0"` TaskID int64 `gorm:"not null;default:0"` TaskQuota int64 `gorm:"not null;default:0"` TaskQuotaTarget int64 `gorm:"not null;default:0"` - Status string `gorm:"type:varchar(16);index;index:idx_billing_settlement_admission,priority:2;index:idx_billing_settlement_status_funding,priority:1;not null;default:'applied'"` + Status string `gorm:"type:varchar(16);index;index:idx_billing_settlement_admission,priority:2;index:idx_billing_settlement_status_funding,priority:1;index:idx_billing_settlement_reconciliation,priority:1;not null;default:'applied'"` Attempts int `gorm:"not null;default:0"` LastError string `gorm:"type:text"` NextAttempt int64 `gorm:"index;not null;default:0"` @@ -92,6 +95,10 @@ type BillingSettlement struct { PreConsumeEffectiveQuota int64 `gorm:"not null;default:0"` EffectPayload string `gorm:"type:text"` EffectStatus string `gorm:"type:varchar(16);index"` + ReconciliationReviewedAt int64 `gorm:"not null;default:0;index:idx_billing_settlement_reconciliation,priority:3"` + ReconciliationReviewedBy int `gorm:"not null;default:0"` + ReconciliationReviewNote string `gorm:"type:text"` + UserBlockingOverride *bool `gorm:"index:idx_billing_settlement_reconciliation,priority:4"` CreatedAt int64 `gorm:"not null"` UpdatedAt int64 `gorm:"index;not null"` } @@ -165,13 +172,58 @@ type BillingSettlementInput struct { PreConsumeEffectiveQuota int64 } -// BillingSettlementBacklogStats is a read-only operational projection of -// positive final settlements that currently block new paid requests. +// BillingSettlementBacklogStats is a read-only operational projection of open +// positive-final-settlement alerts. User admission is evaluated separately by +// the global and per-record blocking policy. type BillingSettlementBacklogStats struct { Count int64 `gorm:"column:record_count"` OldestCreatedAt int64 `gorm:"column:oldest_created_at"` } +// BillingSettlementReconciliationItem is a read-only operator projection. It +// deliberately excludes token keys, effect payloads, request bodies, and any +// other secret-bearing fields. +type BillingSettlementReconciliationItem struct { + ID int64 `json:"id"` + OperationKey string `json:"operation_key"` + Status string `json:"status"` + Source string `json:"source"` + UserID int `json:"user_id"` + SubscriptionID int `json:"subscription_id"` + TokenID int `json:"token_id"` + TaskID int64 `json:"task_id"` + FundingDelta int64 `json:"funding_delta"` + AppliedFundingDelta int64 `json:"applied_funding_delta"` + TokenDelta int64 `json:"token_delta"` + AppliedTokenDelta int64 `json:"applied_token_delta"` + Attempts int `json:"attempts"` + LastError string `json:"last_error"` + NextAttempt int64 `json:"next_attempt"` + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` + ReconciliationReviewedAt int64 `json:"reconciliation_reviewed_at"` + ReconciliationReviewedBy int `json:"reconciliation_reviewed_by"` + ReconciliationReviewNote string `json:"reconciliation_review_note"` + UserBlockingOverride *bool `json:"user_blocking_override"` + RecordBlocksUser bool `json:"record_blocks_user"` + BlocksUser bool `json:"blocks_user"` +} + +type BillingSettlementReconciliationData struct { + TotalCount int64 `json:"total_count"` + PendingCount int64 `json:"pending_count"` + ManualCount int64 `json:"manual_count"` + OpenAlertCount int64 `json:"open_alert_count"` + ReviewedCount int64 `json:"reviewed_count"` + BlockingRecordCount int64 `json:"blocking_record_count"` + BlockedUserCount int64 `json:"blocked_user_count"` + BlockUserByDefault bool `json:"block_user_by_default"` + OldestCreatedAt int64 `json:"oldest_created_at"` + Truncated bool `json:"truncated"` + GeneratedAt int64 `json:"generated_at"` + Items []BillingSettlementReconciliationItem `json:"items"` +} + func unresolvedPositiveFinalizeSettlementScope(db *gorm.DB) *gorm.DB { return db.Model(&BillingSettlement{}). Where("funding_delta > 0"). @@ -179,6 +231,19 @@ func unresolvedPositiveFinalizeSettlementScope(db *gorm.DB) *gorm.DB { Where("operation_key LIKE ?", BillingRequestFinalizeOperationKey("%")) } +func openPositiveFinalizeSettlementAlertScope(db *gorm.DB) *gorm.DB { + return unresolvedPositiveFinalizeSettlementScope(db). + Where("reconciliation_reviewed_at = ?", 0) +} + +func blockingPositiveFinalizeSettlementScope(db *gorm.DB, blockUserByDefault bool) *gorm.DB { + scope := unresolvedPositiveFinalizeSettlementScope(db) + if blockUserByDefault { + return scope.Where("(user_blocking_override IS NULL OR user_blocking_override = ?)", true) + } + return scope.Where("user_blocking_override = ?", true) +} + var billingSettlementRunnerOnce sync.Once var billingSettlementRunnerState struct { sync.Mutex @@ -271,21 +336,156 @@ func observeBillingSettlementBacklog(observedAt time.Time) { } } -// GetUnresolvedPositiveFinalizeSettlementStats returns the global count and -// oldest creation time for positive request-finalize settlements in pending or -// manual state. It is intentionally read-only and portable across all supported -// databases. +// GetUnresolvedPositiveFinalizeSettlementStats returns the count and oldest +// creation time for open (not yet reviewed) positive request-finalize alerts. +// Reviewed rows remain unresolved financial records and are still returned by +// the reconciliation list until the settlement itself reaches applied. func GetUnresolvedPositiveFinalizeSettlementStats() (BillingSettlementBacklogStats, error) { if DB == nil { return BillingSettlementBacklogStats{}, errors.New("database is not initialized") } + return getOpenPositiveFinalizeSettlementAlertStatsDB(DB) +} + +// GetUnresolvedPositiveFinalizeSettlements returns bounded, read-only evidence +// for operators. It never retries, updates, deletes, or otherwise mutates a +// settlement record or any financial balance. +func GetUnresolvedPositiveFinalizeSettlements(limit int) (BillingSettlementReconciliationData, error) { + if DB == nil { + return BillingSettlementReconciliationData{}, errors.New("database is not initialized") + } + if limit <= 0 { + limit = 100 + } + if limit > 200 { + limit = 200 + } + + data := BillingSettlementReconciliationData{ + BlockUserByDefault: billing_reconciliation_setting.BlockUserByDefault(), + GeneratedAt: time.Now().Unix(), + Items: make([]BillingSettlementReconciliationItem, 0), + } + transactionOptions := make([]*sql.TxOptions, 0, 1) + if common.UsingMySQL || common.UsingPostgreSQL { + transactionOptions = append(transactionOptions, &sql.TxOptions{ + Isolation: sql.LevelRepeatableRead, + ReadOnly: true, + }) + } + err := DB.Transaction(func(tx *gorm.DB) error { + stats, err := getAllUnresolvedPositiveFinalizeSettlementStatsDB(tx) + if err != nil { + return err + } + data.TotalCount = stats.Count + data.OldestCreatedAt = stats.OldestCreatedAt + + if err := unresolvedPositiveFinalizeSettlementScope(tx). + Where("status = ?", BillingSettlementStatusPending). + Count(&data.PendingCount).Error; err != nil { + return err + } + if err := unresolvedPositiveFinalizeSettlementScope(tx). + Where("status = ?", BillingSettlementStatusManual). + Count(&data.ManualCount).Error; err != nil { + return err + } + if err := openPositiveFinalizeSettlementAlertScope(tx). + Count(&data.OpenAlertCount).Error; err != nil { + return err + } + data.ReviewedCount = data.TotalCount - data.OpenAlertCount + blockingScope := blockingPositiveFinalizeSettlementScope(tx, data.BlockUserByDefault) + if err := blockingScope.Count(&data.BlockingRecordCount).Error; err != nil { + return err + } + if err := blockingPositiveFinalizeSettlementScope(tx, data.BlockUserByDefault). + Distinct("user_id"). + Count(&data.BlockedUserCount).Error; err != nil { + return err + } + + items := make([]BillingSettlementReconciliationItem, 0, limit+1) + if err := unresolvedPositiveFinalizeSettlementScope(tx). + Select( + "id", "operation_key", "status", "source", "user_id", "subscription_id", "token_id", "task_id", + "funding_delta", "applied_funding_delta", "token_delta", "applied_token_delta", "attempts", "last_error", + "next_attempt", "created_at", "updated_at", "reconciliation_reviewed_at", "reconciliation_reviewed_by", + "reconciliation_review_note", "user_blocking_override", + ). + Order("reconciliation_reviewed_at ASC"). + Order("created_at ASC"). + Order("id ASC"). + Limit(limit + 1). + Scan(&items).Error; err != nil { + return err + } + if len(items) > limit { + data.Truncated = true + items = items[:limit] + } + displayedUserIDs := make([]int, 0, len(items)) + for index := range items { + items[index].LastError = common.SanitizePersistedLogContent( + common.MaskSensitiveInfo(items[index].LastError), + ) + items[index].ReconciliationReviewNote = common.SanitizePersistedLogContent( + common.MaskSensitiveInfo(items[index].ReconciliationReviewNote), + ) + items[index].RecordBlocksUser = billingSettlementBlocksUser( + items[index].UserBlockingOverride, + data.BlockUserByDefault, + ) + if items[index].UserID > 0 { + displayedUserIDs = append(displayedUserIDs, items[index].UserID) + } + } + blockedUsers := make(map[int]struct{}) + if len(displayedUserIDs) > 0 { + blockedUserIDs := make([]int, 0, len(displayedUserIDs)) + if err := blockingPositiveFinalizeSettlementScope(tx, data.BlockUserByDefault). + Where("user_id IN ?", displayedUserIDs). + Distinct("user_id"). + Pluck("user_id", &blockedUserIDs).Error; err != nil { + return err + } + for _, userID := range blockedUserIDs { + blockedUsers[userID] = struct{}{} + } + } + for index := range items { + _, items[index].BlocksUser = blockedUsers[items[index].UserID] + } + data.Items = items + return nil + }, transactionOptions...) + return data, err +} + +func getAllUnresolvedPositiveFinalizeSettlementStatsDB(db *gorm.DB) (BillingSettlementBacklogStats, error) { var stats BillingSettlementBacklogStats - err := unresolvedPositiveFinalizeSettlementScope(DB). + err := unresolvedPositiveFinalizeSettlementScope(db). Select("COUNT(*) AS record_count, COALESCE(MIN(created_at), 0) AS oldest_created_at"). Scan(&stats).Error return stats, err } +func getOpenPositiveFinalizeSettlementAlertStatsDB(db *gorm.DB) (BillingSettlementBacklogStats, error) { + var stats BillingSettlementBacklogStats + err := openPositiveFinalizeSettlementAlertScope(db). + Select("COUNT(*) AS record_count, COALESCE(MIN(created_at), 0) AS oldest_created_at"). + Scan(&stats).Error + return stats, err +} + +func billingSettlementBlocksUser(override *bool, blockUserByDefault bool) bool { + if override != nil { + return *override + } + return blockUserByDefault +} + // HasUnresolvedPositiveFinalizeSettlement reports whether a user has an // upstream-accepted request whose positive final settlement has not completed. // New paid requests must stop before they can consume more service against the @@ -299,8 +499,9 @@ func HasUnresolvedPositiveFinalizeSettlement(userID int) (bool, error) { return false, nil } + blockUserByDefault := billing_reconciliation_setting.BlockUserByDefault() var record BillingSettlement - result := unresolvedPositiveFinalizeSettlementScope(DB). + result := blockingPositiveFinalizeSettlementScope(DB, blockUserByDefault). Select("id"). Where("user_id = ?", userID). Limit(1). @@ -311,6 +512,99 @@ func HasUnresolvedPositiveFinalizeSettlement(userID int) (bool, error) { return result.RowsAffected > 0, nil } +func billingSettlementReviewUpdates(reviewerID int, blockUser bool, note string, reviewedAt time.Time) map[string]interface{} { + return map[string]interface{}{ + "reconciliation_reviewed_at": reviewedAt.Unix(), + "reconciliation_reviewed_by": reviewerID, + "reconciliation_review_note": note, + "user_blocking_override": blockUser, + } +} + +func billingSettlementReviewMatches(record BillingSettlement, reviewerID int, blockUser bool, note string, reviewedAt int64) bool { + return record.ReconciliationReviewedAt == reviewedAt && + record.ReconciliationReviewedBy == reviewerID && + record.ReconciliationReviewNote == note && + record.UserBlockingOverride != nil && + *record.UserBlockingOverride == blockUser +} + +func billingSettlementReviewSnapshotScope(db *gorm.DB, record BillingSettlement) *gorm.DB { + scope := db.Where( + "id = ? AND revision = ? AND reconciliation_reviewed_at = ? AND reconciliation_reviewed_by = ?", + record.ID, + record.Revision, + record.ReconciliationReviewedAt, + record.ReconciliationReviewedBy, + ) + if record.ReconciliationReviewNote == "" { + scope = scope.Where("(reconciliation_review_note = ? OR reconciliation_review_note IS NULL)", "") + } else { + scope = scope.Where("reconciliation_review_note = ?", record.ReconciliationReviewNote) + } + if record.UserBlockingOverride == nil { + return scope.Where("user_blocking_override IS NULL") + } + return scope.Where("user_blocking_override = ?", *record.UserBlockingOverride) +} + +// ReviewBillingSettlement records an administrator's alert disposition without +// changing settlement status, balances, applied deltas, or effect state. +func ReviewBillingSettlement(id int64, reviewerID int, blockUser bool, note string) (BillingSettlement, error) { + if DB == nil { + return BillingSettlement{}, errors.New("database is not initialized") + } + if id <= 0 || reviewerID <= 0 { + return BillingSettlement{}, ErrBillingSettlementReviewConflict + } + + var record BillingSettlement + if err := DB.Select( + "id", + "user_id", + "revision", + "reconciliation_reviewed_at", + "reconciliation_reviewed_by", + "reconciliation_review_note", + "user_blocking_override", + ).First(&record, id).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return BillingSettlement{}, ErrBillingSettlementReviewConflict + } + return BillingSettlement{}, err + } + + reviewedAt := time.Now() + result := billingSettlementReviewSnapshotScope( + unresolvedPositiveFinalizeSettlementScope(DB), + record, + ). + UpdateColumns(billingSettlementReviewUpdates(reviewerID, blockUser, note, reviewedAt)) + if result.Error != nil { + return BillingSettlement{}, result.Error + } + if result.RowsAffected != 1 { + var matching BillingSettlement + err := unresolvedPositiveFinalizeSettlementScope(DB). + Where("id = ?", id). + First(&matching).Error + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return BillingSettlement{}, ErrBillingSettlementReviewConflict + } + return BillingSettlement{}, err + } + if !billingSettlementReviewMatches(matching, reviewerID, blockUser, note, reviewedAt.Unix()) { + return BillingSettlement{}, ErrBillingSettlementReviewConflict + } + return matching, nil + } + if err := DB.First(&record, id).Error; err != nil { + return BillingSettlement{}, err + } + return record, nil +} + func BillingRequestFinalizeOperationKey(requestID string) string { return billingRequestOperationPrefix + requestID + billingRequestFinalizeSuffix } @@ -730,15 +1024,32 @@ func markBillingSettlementFailure(operationKey string, cause error) { if cause != nil { lastError = cause.Error() } - if err := DB.Model(&BillingSettlement{}).Where("id = ? AND status = ?", record.ID, BillingSettlementStatusPending).Updates(map[string]interface{}{ - "attempts": attempts, "last_error": lastError, - "status": status, "next_attempt": nextAttempt, - "updated_at": time.Now().Unix(), "revision": gorm.Expr("revision + ?", 1), - }).Error; err != nil { + resetReview := lastError != record.LastError || status != record.Status + if err := DB.Model(&BillingSettlement{}). + Where("id = ? AND status = ?", record.ID, BillingSettlementStatusPending). + Updates(billingSettlementFailureUpdates(attempts, lastError, status, nextAttempt, time.Now().Unix(), resetReview)).Error; err != nil { common.SysLog("failed to reschedule billing settlement: " + err.Error()) } } +func billingSettlementFailureUpdates(attempts int, lastError string, status string, nextAttempt int64, updatedAt int64, resetReview bool) map[string]interface{} { + updates := map[string]interface{}{ + "attempts": attempts, + "last_error": lastError, + "status": status, + "next_attempt": nextAttempt, + "updated_at": updatedAt, + "revision": gorm.Expr("revision + ?", 1), + } + if resetReview { + updates["reconciliation_reviewed_at"] = 0 + updates["reconciliation_reviewed_by"] = 0 + updates["reconciliation_review_note"] = "" + updates["user_blocking_override"] = nil + } + return updates +} + func processPendingBillingSettlements() { if DB == nil { return diff --git a/model/billing_settlement_regression_test.go b/model/billing_settlement_regression_test.go index b4a89b20..c0f679fd 100644 --- a/model/billing_settlement_regression_test.go +++ b/model/billing_settlement_regression_test.go @@ -4,10 +4,12 @@ import ( "encoding/base64" "errors" "sync" + "sync/atomic" "testing" "time" "github.com/MAX-API-Next/MAX-API/common" + "github.com/MAX-API-Next/MAX-API/setting/billing_reconciliation_setting" "github.com/glebarez/sqlite" "github.com/go-webauthn/webauthn/protocol" "github.com/go-webauthn/webauthn/webauthn" @@ -84,8 +86,10 @@ func TestConcurrentBillingPreConsumeSelectionAllowsOnlyOneFundingSource(t *testi func TestHasUnresolvedPositiveFinalizeSettlement(t *testing.T) { setupUserUpdateTestState(t) + setBillingReconciliationBlockDefaultForTest(t, true) require.True(t, DB.Migrator().HasIndex(&BillingSettlement{}, "idx_billing_settlement_admission")) require.True(t, DB.Migrator().HasIndex(&BillingSettlement{}, "idx_billing_settlement_status_funding")) + require.True(t, DB.Migrator().HasIndex(&BillingSettlement{}, "idx_billing_settlement_reconciliation")) const blockingUserID = 941 const nonBlockingUserID = 942 @@ -95,7 +99,8 @@ func TestHasUnresolvedPositiveFinalizeSettlement(t *testing.T) { { OperationKey: "request:pending-finalize:finalize", Source: BillingSettlementSourceWallet, UserID: blockingUserID, FundingDelta: 10, TokenDelta: 10, - Status: BillingSettlementStatusPending, NextAttempt: now, CreatedAt: oldest, UpdatedAt: now, Revision: 1, + Status: BillingSettlementStatusPending, LastError: "upstream https://api.example.com/v1?api_key=secret failed", + NextAttempt: now, CreatedAt: oldest, UpdatedAt: now, Revision: 1, }, { OperationKey: "request:manual-finalize:finalize", Source: BillingSettlementSourceWallet, @@ -129,6 +134,26 @@ func TestHasUnresolvedPositiveFinalizeSettlement(t *testing.T) { assert.EqualValues(t, 2, stats.Count) assert.Equal(t, oldest, stats.OldestCreatedAt) + reconciliation, err := GetUnresolvedPositiveFinalizeSettlements(1) + require.NoError(t, err) + assert.EqualValues(t, 2, reconciliation.TotalCount) + assert.EqualValues(t, 1, reconciliation.PendingCount) + assert.EqualValues(t, 1, reconciliation.ManualCount) + assert.EqualValues(t, 2, reconciliation.OpenAlertCount) + assert.Zero(t, reconciliation.ReviewedCount) + assert.EqualValues(t, 2, reconciliation.BlockingRecordCount) + assert.EqualValues(t, 1, reconciliation.BlockedUserCount) + assert.True(t, reconciliation.BlockUserByDefault) + assert.Equal(t, oldest, reconciliation.OldestCreatedAt) + assert.True(t, reconciliation.Truncated) + require.Len(t, reconciliation.Items, 1) + assert.Equal(t, "request:pending-finalize:finalize", reconciliation.Items[0].OperationKey) + assert.Equal(t, BillingSettlementStatusPending, reconciliation.Items[0].Status) + assert.EqualValues(t, 10, reconciliation.Items[0].FundingDelta) + assert.NotContains(t, reconciliation.Items[0].LastError, "api.example.com") + assert.NotContains(t, reconciliation.Items[0].LastError, "secret") + assert.Contains(t, reconciliation.Items[0].LastError, "***") + require.NoError(t, DB.Where("user_id = ?", blockingUserID).Delete(&BillingSettlement{}).Error) stats, err = GetUnresolvedPositiveFinalizeSettlementStats() require.NoError(t, err) @@ -136,6 +161,406 @@ func TestHasUnresolvedPositiveFinalizeSettlement(t *testing.T) { assert.Zero(t, stats.OldestCreatedAt) } +func TestBillingSettlementBlockingPolicyAndReviewAreIndependentFromFinancialState(t *testing.T) { + setupUserUpdateTestState(t) + setBillingReconciliationBlockDefaultForTest(t, false) + + now := time.Now().Unix() + block := true + allow := false + records := []BillingSettlement{ + { + OperationKey: "request:inherit-policy:finalize", Source: BillingSettlementSourceWallet, + UserID: 951, FundingDelta: 10, AppliedFundingDelta: 2, TokenDelta: 10, AppliedTokenDelta: 2, + Status: BillingSettlementStatusPending, CreatedAt: now - 30, UpdatedAt: now, Revision: 1, + }, + { + OperationKey: "request:force-block:finalize", Source: BillingSettlementSourceWallet, + UserID: 952, FundingDelta: 11, TokenDelta: 11, Status: BillingSettlementStatusManual, + UserBlockingOverride: &block, CreatedAt: now - 20, UpdatedAt: now, Revision: 1, + }, + { + OperationKey: "request:force-allow:finalize", Source: BillingSettlementSourceWallet, + UserID: 953, FundingDelta: 12, TokenDelta: 12, Status: BillingSettlementStatusPending, + UserBlockingOverride: &allow, CreatedAt: now - 10, UpdatedAt: now, Revision: 1, + }, + { + OperationKey: "request:force-allow-shared-user:finalize", Source: BillingSettlementSourceWallet, + UserID: 954, FundingDelta: 13, TokenDelta: 13, Status: BillingSettlementStatusPending, + UserBlockingOverride: &allow, CreatedAt: now - 5, UpdatedAt: now, Revision: 1, + }, + { + OperationKey: "request:force-block-shared-user:finalize", Source: BillingSettlementSourceWallet, + UserID: 954, FundingDelta: 14, TokenDelta: 14, Status: BillingSettlementStatusManual, + UserBlockingOverride: &block, CreatedAt: now - 1, UpdatedAt: now, Revision: 1, + }, + } + require.NoError(t, DB.Create(&records).Error) + + blocked, err := HasUnresolvedPositiveFinalizeSettlement(951) + require.NoError(t, err) + assert.False(t, blocked, "nil override must inherit the disabled global policy") + blocked, err = HasUnresolvedPositiveFinalizeSettlement(952) + require.NoError(t, err) + assert.True(t, blocked, "an explicit per-record block must override the global policy") + blocked, err = HasUnresolvedPositiveFinalizeSettlement(953) + require.NoError(t, err) + assert.False(t, blocked, "an explicit per-record allow must remain non-blocking") + blocked, err = HasUnresolvedPositiveFinalizeSettlement(954) + require.NoError(t, err) + assert.True(t, blocked, "any blocking record must keep the same user blocked") + + reviewed, err := ReviewBillingSettlement(records[0].ID, 7001, false, "Verified provider usage against the invoice") + require.NoError(t, err) + assert.Equal(t, records[0].ID, reviewed.ID) + assert.Equal(t, records[0].UserID, reviewed.UserID) + assert.Equal(t, BillingSettlementStatusPending, reviewed.Status) + assert.EqualValues(t, 10, reviewed.FundingDelta) + assert.EqualValues(t, 2, reviewed.AppliedFundingDelta) + assert.EqualValues(t, 10, reviewed.TokenDelta) + assert.EqualValues(t, 2, reviewed.AppliedTokenDelta) + assert.Positive(t, reviewed.ReconciliationReviewedAt) + assert.Equal(t, 7001, reviewed.ReconciliationReviewedBy) + assert.Equal(t, "Verified provider usage against the invoice", reviewed.ReconciliationReviewNote) + require.NotNil(t, reviewed.UserBlockingOverride) + assert.False(t, *reviewed.UserBlockingOverride, "explicit false must be durably persisted") + assert.Equal(t, records[0].UpdatedAt, reviewed.UpdatedAt, "review metadata must not replace the financial update timestamp") + assert.Equal(t, records[0].Revision, reviewed.Revision, "review metadata must not advance the financial settlement revision") + + stats, err := GetUnresolvedPositiveFinalizeSettlementStats() + require.NoError(t, err) + assert.EqualValues(t, 4, stats.Count, "reviewed records must leave the open-alert projection") + + reconciliation, err := GetUnresolvedPositiveFinalizeSettlements(100) + require.NoError(t, err) + assert.EqualValues(t, 5, reconciliation.TotalCount, "review does not complete or delete a settlement") + assert.EqualValues(t, 4, reconciliation.OpenAlertCount) + assert.EqualValues(t, 1, reconciliation.ReviewedCount) + assert.EqualValues(t, 2, reconciliation.BlockingRecordCount) + assert.EqualValues(t, 2, reconciliation.BlockedUserCount) + assert.False(t, reconciliation.BlockUserByDefault) + require.Len(t, reconciliation.Items, 5) + + var reviewedItem *BillingSettlementReconciliationItem + var sharedAllowItem *BillingSettlementReconciliationItem + var sharedBlockItem *BillingSettlementReconciliationItem + for index := range reconciliation.Items { + switch reconciliation.Items[index].ID { + case records[0].ID: + reviewedItem = &reconciliation.Items[index] + case records[3].ID: + sharedAllowItem = &reconciliation.Items[index] + case records[4].ID: + sharedBlockItem = &reconciliation.Items[index] + } + } + require.NotNil(t, reviewedItem) + assert.False(t, reviewedItem.RecordBlocksUser) + assert.False(t, reviewedItem.BlocksUser) + assert.Equal(t, 7001, reviewedItem.ReconciliationReviewedBy) + assert.Equal(t, "Verified provider usage against the invoice", reviewedItem.ReconciliationReviewNote) + require.NotNil(t, sharedAllowItem) + assert.False(t, sharedAllowItem.RecordBlocksUser, "the row-level decision must remain allow") + assert.True(t, sharedAllowItem.BlocksUser, "the user remains blocked by another unresolved record") + require.NotNil(t, sharedBlockItem) + assert.True(t, sharedBlockItem.RecordBlocksUser) + assert.True(t, sharedBlockItem.BlocksUser) +} + +func TestBillingSettlementBlockingScopeDoesNotLeakAcrossUsers(t *testing.T) { + setupUserUpdateTestState(t) + setBillingReconciliationBlockDefaultForTest(t, true) + + block := true + record := BillingSettlement{ + OperationKey: "request:cross-user-blocking:finalize", Source: BillingSettlementSourceWallet, + UserID: 961, FundingDelta: 10, TokenDelta: 10, Status: BillingSettlementStatusPending, + UserBlockingOverride: &block, CreatedAt: time.Now().Unix(), UpdatedAt: time.Now().Unix(), Revision: 1, + } + require.NoError(t, DB.Create(&record).Error) + + blocked, err := HasUnresolvedPositiveFinalizeSettlement(record.UserID) + require.NoError(t, err) + assert.True(t, blocked) + + blocked, err = HasUnresolvedPositiveFinalizeSettlement(962) + require.NoError(t, err) + assert.False(t, blocked, "another user's explicit block must not leak across the user_id predicate") +} + +func TestBillingSettlementFailureReopensReviewedRecord(t *testing.T) { + setupUserUpdateTestState(t) + setBillingReconciliationBlockDefaultForTest(t, true) + + now := time.Now().Unix() + allow := false + record := BillingSettlement{ + OperationKey: "request:reviewed-retry-failure:finalize", Source: BillingSettlementSourceWallet, + UserID: 963, FundingDelta: 10, AppliedFundingDelta: 2, TokenDelta: 10, AppliedTokenDelta: 2, + Status: BillingSettlementStatusPending, Attempts: 1, LastError: "previous failure", NextAttempt: now - 1, + ReconciliationReviewedAt: now - 60, ReconciliationReviewedBy: 7005, + ReconciliationReviewNote: "Reviewed previous failure evidence", UserBlockingOverride: &allow, + CreatedAt: now - 120, UpdatedAt: now - 60, Revision: 7, + } + require.NoError(t, DB.Create(&record).Error) + + stats, err := GetUnresolvedPositiveFinalizeSettlementStats() + require.NoError(t, err) + assert.Zero(t, stats.Count) + blocked, err := HasUnresolvedPositiveFinalizeSettlement(record.UserID) + require.NoError(t, err) + assert.False(t, blocked, "the reviewed allow decision is active before new evidence arrives") + + markBillingSettlementFailure(record.OperationKey, errors.New("previous failure")) + + var stored BillingSettlement + require.NoError(t, DB.First(&stored, record.ID).Error) + assert.Equal(t, 2, stored.Attempts) + assert.Equal(t, "previous failure", stored.LastError) + assert.Equal(t, BillingSettlementStatusPending, stored.Status) + assert.EqualValues(t, record.ReconciliationReviewedAt, stored.ReconciliationReviewedAt) + assert.Equal(t, record.ReconciliationReviewedBy, stored.ReconciliationReviewedBy) + assert.Equal(t, record.ReconciliationReviewNote, stored.ReconciliationReviewNote) + require.NotNil(t, stored.UserBlockingOverride) + assert.False(t, *stored.UserBlockingOverride) + assert.EqualValues(t, 8, stored.Revision) + + stats, err = GetUnresolvedPositiveFinalizeSettlementStats() + require.NoError(t, err) + assert.Zero(t, stats.Count, "unchanged failure evidence must preserve the administrator review") + blocked, err = HasUnresolvedPositiveFinalizeSettlement(record.UserID) + require.NoError(t, err) + assert.False(t, blocked, "unchanged failure evidence must preserve the explicit allow decision") + + markBillingSettlementFailure(record.OperationKey, errors.New("new retry failure")) + + require.NoError(t, DB.First(&stored, record.ID).Error) + assert.Equal(t, 3, stored.Attempts) + assert.Equal(t, "new retry failure", stored.LastError) + assert.Equal(t, BillingSettlementStatusPending, stored.Status) + assert.GreaterOrEqual(t, stored.NextAttempt, now) + assert.Zero(t, stored.ReconciliationReviewedAt) + assert.Zero(t, stored.ReconciliationReviewedBy) + assert.Empty(t, stored.ReconciliationReviewNote) + assert.Nil(t, stored.UserBlockingOverride) + assert.EqualValues(t, 10, stored.FundingDelta) + assert.EqualValues(t, 2, stored.AppliedFundingDelta) + assert.EqualValues(t, 10, stored.TokenDelta) + assert.EqualValues(t, 2, stored.AppliedTokenDelta) + assert.EqualValues(t, 9, stored.Revision) + + stats, err = GetUnresolvedPositiveFinalizeSettlementStats() + require.NoError(t, err) + assert.EqualValues(t, 1, stats.Count, "new failure evidence must reopen the operational alert") + blocked, err = HasUnresolvedPositiveFinalizeSettlement(record.UserID) + require.NoError(t, err) + assert.True(t, blocked, "reopened records must inherit the fail-closed global policy") +} + +func TestBillingSettlementFailureReopensReviewedRecordWhenStatusBecomesManual(t *testing.T) { + setupUserUpdateTestState(t) + + now := time.Now().Unix() + allow := false + record := BillingSettlement{ + OperationKey: "request:reviewed-status-change:finalize", Source: BillingSettlementSourceWallet, + UserID: 967, FundingDelta: 10, TokenDelta: 10, + Status: BillingSettlementStatusPending, Attempts: 1, LastError: "unchanged failure", + ManualOnFailure: true, ReconciliationReviewedAt: now - 60, ReconciliationReviewedBy: 7008, + ReconciliationReviewNote: "Reviewed while retry remained pending", UserBlockingOverride: &allow, + CreatedAt: now - 120, UpdatedAt: now - 60, Revision: 3, + } + require.NoError(t, DB.Create(&record).Error) + + markBillingSettlementFailure(record.OperationKey, errors.New("unchanged failure")) + + var stored BillingSettlement + require.NoError(t, DB.First(&stored, record.ID).Error) + assert.Equal(t, BillingSettlementStatusManual, stored.Status) + assert.Equal(t, "unchanged failure", stored.LastError) + assert.Zero(t, stored.ReconciliationReviewedAt) + assert.Zero(t, stored.ReconciliationReviewedBy) + assert.Empty(t, stored.ReconciliationReviewNote) + assert.Nil(t, stored.UserBlockingOverride) + assert.EqualValues(t, 4, stored.Revision) +} + +func TestReviewBillingSettlementRejectsAlreadyAppliedRecord(t *testing.T) { + setupUserUpdateTestState(t) + now := time.Now().Unix() + record := BillingSettlement{ + OperationKey: "request:review-race:finalize", Source: BillingSettlementSourceWallet, + UserID: 954, FundingDelta: 10, AppliedFundingDelta: 10, TokenDelta: 10, AppliedTokenDelta: 10, + Status: BillingSettlementStatusApplied, CreatedAt: now, UpdatedAt: now, Revision: 2, + } + require.NoError(t, DB.Create(&record).Error) + + _, err := ReviewBillingSettlement(record.ID, 7002, false, "Already reconciled elsewhere") + + assert.ErrorIs(t, err, ErrBillingSettlementReviewConflict) + var stored BillingSettlement + require.NoError(t, DB.First(&stored, record.ID).Error) + assert.Zero(t, stored.ReconciliationReviewedAt) + assert.Empty(t, stored.ReconciliationReviewNote) +} + +func TestReviewBillingSettlementRejectsStaleFailureEvidence(t *testing.T) { + setupUserUpdateTestState(t) + setBillingReconciliationBlockDefaultForTest(t, true) + + now := time.Now().Unix() + record := BillingSettlement{ + OperationKey: "request:stale-review-race:finalize", Source: BillingSettlementSourceWallet, + UserID: 964, FundingDelta: 10, TokenDelta: 10, Status: BillingSettlementStatusPending, + Attempts: 1, LastError: "previous failure", NextAttempt: now - 1, + CreatedAt: now - 60, UpdatedAt: now - 30, Revision: 4, + } + require.NoError(t, DB.Create(&record).Error) + + callbackName := "test:billing-settlement-review-stale-failure" + var failureInjected atomic.Bool + require.NoError(t, DB.Callback().Query().After("gorm:query").Register(callbackName, func(tx *gorm.DB) { + if tx.Statement != nil && tx.Statement.Schema != nil && tx.Statement.Schema.Name == "BillingSettlement" && failureInjected.CompareAndSwap(false, true) { + markBillingSettlementFailure(record.OperationKey, errors.New("new failure evidence")) + } + })) + t.Cleanup(func() { + _ = DB.Callback().Query().Remove(callbackName) + }) + + _, err := ReviewBillingSettlement(record.ID, 7006, false, "Reviewed superseded evidence") + + assert.ErrorIs(t, err, ErrBillingSettlementReviewConflict) + var stored BillingSettlement + require.NoError(t, DB.First(&stored, record.ID).Error) + assert.Equal(t, 2, stored.Attempts) + assert.Equal(t, "new failure evidence", stored.LastError) + assert.Zero(t, stored.ReconciliationReviewedAt) + assert.Zero(t, stored.ReconciliationReviewedBy) + assert.Empty(t, stored.ReconciliationReviewNote) + assert.Nil(t, stored.UserBlockingOverride) + assert.EqualValues(t, 5, stored.Revision) + + stats, statsErr := GetUnresolvedPositiveFinalizeSettlementStats() + require.NoError(t, statsErr) + assert.EqualValues(t, 1, stats.Count, "the newer failure must remain visible for administrator review") +} + +func TestReviewBillingSettlementAllowsEditingCurrentReview(t *testing.T) { + setupUserUpdateTestState(t) + + now := time.Now().Unix() + allow := false + record := BillingSettlement{ + OperationKey: "request:edit-current-review:finalize", Source: BillingSettlementSourceWallet, + UserID: 965, FundingDelta: 10, TokenDelta: 10, Status: BillingSettlementStatusPending, + ReconciliationReviewedAt: now - 60, ReconciliationReviewedBy: 7007, + ReconciliationReviewNote: "Initial review", UserBlockingOverride: &allow, + CreatedAt: now - 120, UpdatedAt: now - 60, Revision: 3, + } + require.NoError(t, DB.Create(&record).Error) + + reviewed, err := ReviewBillingSettlement(record.ID, 7008, true, "Updated review after checking more evidence") + + require.NoError(t, err) + assert.Equal(t, 7008, reviewed.ReconciliationReviewedBy) + assert.Equal(t, "Updated review after checking more evidence", reviewed.ReconciliationReviewNote) + require.NotNil(t, reviewed.UserBlockingOverride) + assert.True(t, *reviewed.UserBlockingOverride) + assert.EqualValues(t, record.Revision, reviewed.Revision, "review metadata must remain independent from financial revision") +} + +func TestReviewBillingSettlementAcceptsLegacyNullReviewNote(t *testing.T) { + setupUserUpdateTestState(t) + + now := time.Now().Unix() + record := BillingSettlement{ + OperationKey: "request:legacy-null-review-note:finalize", Source: BillingSettlementSourceWallet, + UserID: 966, FundingDelta: 10, TokenDelta: 10, Status: BillingSettlementStatusPending, + CreatedAt: now - 60, UpdatedAt: now - 30, Revision: 2, + } + require.NoError(t, DB.Create(&record).Error) + require.NoError(t, DB.Exec( + "UPDATE billing_settlements SET reconciliation_review_note = NULL WHERE id = ?", + record.ID, + ).Error) + + reviewed, err := ReviewBillingSettlement(record.ID, 7009, false, "Reviewed legacy reconciliation record") + + require.NoError(t, err) + assert.Equal(t, 7009, reviewed.ReconciliationReviewedBy) + assert.Equal(t, "Reviewed legacy reconciliation record", reviewed.ReconciliationReviewNote) + require.NotNil(t, reviewed.UserBlockingOverride) + assert.False(t, *reviewed.UserBlockingOverride) +} + +func TestReviewBillingSettlementAcceptsMatchingUpdateWhenDriverReportsZeroRows(t *testing.T) { + setupUserUpdateTestState(t) + now := time.Now().Unix() + record := BillingSettlement{ + OperationKey: "request:review-same-value:finalize", Source: BillingSettlementSourceWallet, + UserID: 955, FundingDelta: 10, AppliedFundingDelta: 2, TokenDelta: 10, AppliedTokenDelta: 2, + Status: BillingSettlementStatusPending, CreatedAt: now, UpdatedAt: now, Revision: 1, + } + require.NoError(t, DB.Create(&record).Error) + + callbackName := "test:billing-settlement-review-zero-rows" + require.NoError(t, DB.Callback().Update().After("gorm:update").Register(callbackName, func(tx *gorm.DB) { + if tx.Statement != nil && tx.Statement.Schema != nil && tx.Statement.Schema.Name == "BillingSettlement" { + tx.RowsAffected = 0 + } + })) + t.Cleanup(func() { + _ = DB.Callback().Update().Remove(callbackName) + }) + + reviewed, err := ReviewBillingSettlement(record.ID, 7004, false, "Verified duplicate review submission") + + require.NoError(t, err) + assert.Equal(t, record.ID, reviewed.ID) + assert.Equal(t, 7004, reviewed.ReconciliationReviewedBy) + assert.Equal(t, "Verified duplicate review submission", reviewed.ReconciliationReviewNote) + require.NotNil(t, reviewed.UserBlockingOverride) + assert.False(t, *reviewed.UserBlockingOverride) +} + +func TestBillingSettlementReviewUpdateMapUsesSchemaColumns(t *testing.T) { + parsed, err := schema.Parse(&BillingSettlement{}, &sync.Map{}, schema.NamingStrategy{}) + require.NoError(t, err) + + updateMaps := []map[string]interface{}{ + billingSettlementReviewUpdates(7003, false, "reviewed", time.Unix(123, 0)), + billingSettlementFailureUpdates(2, "failed", BillingSettlementStatusPending, 456, 123, false), + billingSettlementFailureUpdates(2, "failed", BillingSettlementStatusManual, 0, 123, true), + } + for _, updates := range updateMaps { + for key := range updates { + assert.NotNilf(t, parsed.LookUpField(key), "billing settlement update key %q must map to a BillingSettlement column", key) + } + } +} + +func setBillingReconciliationBlockDefaultForTest(t *testing.T, enabled bool) { + t.Helper() + key := billing_reconciliation_setting.OptionKeyBlockUserByDefault + common.OptionMapRWMutex.Lock() + if common.OptionMap == nil { + common.OptionMap = make(map[string]string) + } + original, existed := common.OptionMap[key] + common.OptionMap[key] = map[bool]string{true: "true", false: "false"}[enabled] + common.OptionMapRWMutex.Unlock() + t.Cleanup(func() { + common.OptionMapRWMutex.Lock() + defer common.OptionMapRWMutex.Unlock() + if existed { + common.OptionMap[key] = original + } else { + delete(common.OptionMap, key) + } + }) +} + func TestBillingRequestFinalizeOperationKey(t *testing.T) { assert.Equal(t, "request:example-request:finalize", BillingRequestFinalizeOperationKey("example-request")) assert.Equal(t, "request:%:finalize", BillingRequestFinalizeOperationKey("%")) diff --git a/router/api-router.go b/router/api-router.go index 99c6efd1..0454d2b8 100644 --- a/router/api-router.go +++ b/router/api-router.go @@ -234,6 +234,9 @@ func SetApiRouter(router *gin.Engine) { smartOpsRoute.Use(middleware.AdminAuth()) { smartOpsRoute.GET("/alerts", controller.GetSmartOpsAlerts) + smartOpsRoute.GET("/billing-settlements", middleware.DisableCache(), controller.GetBillingSettlementReconciliation) + smartOpsRoute.PUT("/billing-settlements/blocking-policy", middleware.RootAuth(), middleware.CriticalRateLimit(), middleware.UserCriticalRateLimit(), middleware.DisableCache(), anonymousRequestBodyLimit, controller.UpdateBillingSettlementBlockingPolicy) + smartOpsRoute.POST("/billing-settlements/:id/review", middleware.CriticalRateLimit(), middleware.UserCriticalRateLimit(), middleware.DisableCache(), anonymousRequestBodyLimit, controller.ReviewBillingSettlement) smartOpsRoute.GET("/channel-performance", controller.GetChannelPerformance) smartOpsRoute.GET("/channel-performance/detail", controller.GetChannelPerformanceDetail) smartOpsRoute.GET("/model-performance", controller.GetModelPerformance) diff --git a/router/api_router_test.go b/router/api_router_test.go index ef38a19c..7a399d7b 100644 --- a/router/api_router_test.go +++ b/router/api_router_test.go @@ -12,6 +12,7 @@ import ( "github.com/MAX-API-Next/MAX-API/common" "github.com/MAX-API-Next/MAX-API/model" + "github.com/MAX-API-Next/MAX-API/setting/billing_reconciliation_setting" "github.com/gin-contrib/sessions" "github.com/gin-contrib/sessions/cookie" "github.com/gin-gonic/gin" @@ -140,6 +141,126 @@ func TestHealthRoutesAreRegistered(t *testing.T) { } } +func TestBillingSettlementBlockingPolicyRequiresRootRole(t *testing.T) { + gin.SetMode(gin.TestMode) + testRun := atomic.AddInt32(&universalVerifyRateLimitTestRun, 1) + + oldDB := model.DB + oldLogDB := model.LOG_DB + oldRedisEnabled := common.RedisEnabled + oldMemoryCacheEnabled := common.MemoryCacheEnabled + oldGlobalAPIRateLimitEnable := common.GlobalApiRateLimitEnable + oldCriticalRateLimitEnable := common.CriticalRateLimitEnable + t.Cleanup(func() { + model.DB = oldDB + model.LOG_DB = oldLogDB + common.RedisEnabled = oldRedisEnabled + common.MemoryCacheEnabled = oldMemoryCacheEnabled + common.GlobalApiRateLimitEnable = oldGlobalAPIRateLimitEnable + common.CriticalRateLimitEnable = oldCriticalRateLimitEnable + }) + common.RedisEnabled = false + common.MemoryCacheEnabled = false + common.GlobalApiRateLimitEnable = false + common.CriticalRateLimitEnable = false + + dsn := fmt.Sprintf("file:billing_policy_root_%d?mode=memory&cache=shared", testRun) + db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{}) + require.NoError(t, err) + t.Cleanup(func() { + if sqlDB, dbErr := db.DB(); dbErr == nil { + _ = sqlDB.Close() + } + }) + require.NoError(t, db.AutoMigrate(&model.User{}, &model.Option{}, &model.Log{})) + model.DB = db + model.LOG_DB = db + + key := billing_reconciliation_setting.OptionKeyBlockUserByDefault + common.OptionMapRWMutex.Lock() + if common.OptionMap == nil { + common.OptionMap = make(map[string]string) + } + originalPolicy, policyExisted := common.OptionMap[key] + common.OptionMap[key] = "true" + common.OptionMapRWMutex.Unlock() + t.Cleanup(func() { + common.OptionMapRWMutex.Lock() + defer common.OptionMapRWMutex.Unlock() + if policyExisted { + common.OptionMap[key] = originalPolicy + } else { + delete(common.OptionMap, key) + } + }) + + admin := model.User{ + Id: 1500000 + int(testRun), Username: fmt.Sprintf("billing-policy-admin-%d", testRun), + AffCode: fmt.Sprintf("billing-policy-admin-aff-%d", testRun), + Password: "hashed-password", Role: common.RoleAdminUser, + Status: common.UserStatusEnabled, Group: "default", + } + root := model.User{ + Id: admin.Id + 1, Username: fmt.Sprintf("billing-policy-root-%d", testRun), + AffCode: fmt.Sprintf("billing-policy-root-aff-%d", testRun), + Password: "hashed-password", Role: common.RoleRootUser, + Status: common.UserStatusEnabled, Group: "default", + } + require.NoError(t, db.Create(&admin).Error) + require.NoError(t, db.Create(&root).Error) + + engine := gin.New() + engine.Use(sessions.Sessions("session", cookie.NewStore([]byte("billing-policy-root-session")))) + engine.GET("/test/login/:id", func(c *gin.Context) { + id := admin.Id + user := admin + if c.Param("id") == fmt.Sprintf("%d", root.Id) { + id = root.Id + user = root + } + session := sessions.Default(c) + session.Set("id", id) + session.Set("username", user.Username) + session.Set("role", user.Role) + session.Set("status", user.Status) + session.Set("group", user.Group) + require.NoError(t, session.Save()) + c.Status(http.StatusNoContent) + }) + SetApiRouter(engine) + + login := func(userID int) []*http.Cookie { + recorder := httptest.NewRecorder() + engine.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, fmt.Sprintf("/test/login/%d", userID), nil)) + require.Equal(t, http.StatusNoContent, recorder.Code) + return recorder.Result().Cookies() + } + updatePolicy := func(cookies []*http.Cookie) *httptest.ResponseRecorder { + request := httptest.NewRequest( + http.MethodPut, + "/api/smart-ops/billing-settlements/blocking-policy", + strings.NewReader(`{"block_user_by_default":false}`), + ) + request.Header.Set("Content-Type", "application/json") + for _, sessionCookie := range cookies { + request.AddCookie(sessionCookie) + } + recorder := httptest.NewRecorder() + engine.ServeHTTP(recorder, request) + return recorder + } + + adminResponse := updatePolicy(login(admin.Id)) + require.Equal(t, http.StatusOK, adminResponse.Code) + require.Contains(t, adminResponse.Body.String(), `"success":false`) + require.True(t, billing_reconciliation_setting.BlockUserByDefault()) + + rootResponse := updatePolicy(login(root.Id)) + require.Equal(t, http.StatusOK, rootResponse.Code) + require.Contains(t, rootResponse.Body.String(), `"success":true`) + require.False(t, billing_reconciliation_setting.BlockUserByDefault()) +} + func TestUniversalVerifyRateLimitFollowsUserAcrossIPs(t *testing.T) { gin.SetMode(gin.TestMode) testRun := atomic.AddInt32(&universalVerifyRateLimitTestRun, 1) diff --git a/service/billing_session.go b/service/billing_session.go index 0798c588..ac1f5538 100644 --- a/service/billing_session.go +++ b/service/billing_session.go @@ -64,9 +64,11 @@ var billingSettlementBacklogAlertState struct { lastCount int64 oldestCreatedAt int64 lastNotifiedAt time.Time + lastObservedAt time.Time } var billingSettlementBacklogAlertNotificationSender = enqueueSmartOpsAlertNotification +var billingSettlementBacklogAlertProjector = projectSmartOpsActiveAlert var billingFundingOutcomeUnknownAlertNotificationSender = enqueueSmartOpsAlertNotification func init() { @@ -74,11 +76,21 @@ func init() { } func observeBillingSettlementBacklog(stats model.BillingSettlementBacklogStats, observedAt time.Time) { + updateBillingSettlementBacklogAlert(stats, observedAt, true) +} + +func updateBillingSettlementBacklogAlert(stats model.BillingSettlementBacklogStats, observedAt time.Time, notify bool) { if observedAt.IsZero() { observedAt = time.Now() } billingSettlementBacklogAlertState.Lock() + if !billingSettlementBacklogAlertState.lastObservedAt.IsZero() && + !observedAt.After(billingSettlementBacklogAlertState.lastObservedAt) { + billingSettlementBacklogAlertState.Unlock() + return + } + billingSettlementBacklogAlertState.lastObservedAt = observedAt wasActive := billingSettlementBacklogAlertState.active previousCount := billingSettlementBacklogAlertState.lastCount previousOldestCreatedAt := billingSettlementBacklogAlertState.oldestCreatedAt @@ -87,19 +99,24 @@ func observeBillingSettlementBacklog(stats model.BillingSettlementBacklogStats, billingSettlementBacklogAlertState.lastCount = 0 billingSettlementBacklogAlertState.oldestCreatedAt = 0 billingSettlementBacklogAlertState.lastNotifiedAt = time.Time{} + resolvedAlert := SmartOpsAlert{ + Key: "billing_settlement_backlog", + Status: smartOpsAlertStatusResolved, + Severity: smartOpsAlertSeverityWarning, + Component: "billing", + Node: smartOpsAlertNodeName(), + CurrentValue: 0, + Threshold: 0, + ObservedAt: observedAt, + Message: fmt.Sprintf( + "尚未核对的正向最终计费告警已全部关闭，此前共 %d 条。财务结算记录仍可能保持 pending/manual，请继续在对账列表中处理。", + previousCount, + ), + } + billingSettlementBacklogAlertProjector(resolvedAlert) billingSettlementBacklogAlertState.Unlock() - if wasActive { - billingSettlementBacklogAlertNotificationSender(SmartOpsAlert{ - Key: "billing_settlement_backlog", - Status: smartOpsAlertStatusResolved, - Severity: smartOpsAlertSeverityWarning, - Component: "billing", - Node: smartOpsAlertNodeName(), - CurrentValue: 0, - Threshold: 0, - ObservedAt: observedAt, - Message: fmt.Sprintf("未完成的正向最终计费对账已清空，此前共 %d 条。", previousCount), - }) + if wasActive && notify { + billingSettlementBacklogAlertNotificationSender(resolvedAlert) } return } @@ -119,15 +136,10 @@ func observeBillingSettlementBacklog(stats model.BillingSettlementBacklogStats, billingSettlementBacklogAlertState.active = true billingSettlementBacklogAlertState.lastCount = stats.Count billingSettlementBacklogAlertState.oldestCreatedAt = stats.OldestCreatedAt - if shouldNotify { + if shouldNotify && notify { billingSettlementBacklogAlertState.lastNotifiedAt = observedAt } - billingSettlementBacklogAlertState.Unlock() - if !shouldNotify { - return - } - - billingSettlementBacklogAlertNotificationSender(SmartOpsAlert{ + alert := SmartOpsAlert{ Key: "billing_settlement_backlog", Status: smartOpsAlertStatusFiring, Severity: smartOpsAlertSeverityWarning, @@ -137,11 +149,25 @@ func observeBillingSettlementBacklog(stats model.BillingSettlementBacklogStats, Threshold: oldestAge.Seconds(), ObservedAt: observedAt, Message: fmt.Sprintf( - "存在 %d 条未完成的正向最终计费对账，最早已等待 %s。相关用户的新付费请求会被阻止，请尽快处理 pending/manual 记录。", + "存在 %d 条尚未核对的正向最终计费对账，最早已等待 %s。是否阻止相关用户的新付费请求由管理员配置和单条核对策略决定，请尽快处理 pending/manual 记录。", stats.Count, oldestAge.Round(time.Second), ), - }) + } + billingSettlementBacklogAlertProjector(alert) + billingSettlementBacklogAlertState.Unlock() + if shouldNotify && notify { + billingSettlementBacklogAlertNotificationSender(alert) + } +} + +func refreshBillingSettlementBacklogAfterReview() { + stats, err := model.GetUnresolvedPositiveFinalizeSettlementStats() + if err != nil { + common.SysLog(fmt.Sprintf("failed to refresh billing settlement backlog after administrator review: %s", err.Error())) + return + } + updateBillingSettlementBacklogAlert(stats, time.Now(), false) } func (s *BillingSession) notifyFundingOutcomeUnknown(requested, applied int64) { diff --git a/service/billing_session_test.go b/service/billing_session_test.go index d552b659..632d0cca 100644 --- a/service/billing_session_test.go +++ b/service/billing_session_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "sync" "testing" "time" @@ -39,34 +40,56 @@ type uncertainFundingSource struct { err error } -func TestBillingSettlementBacklogAlertReportsCountAgeAndRecovery(t *testing.T) { - originalSender := billingSettlementBacklogAlertNotificationSender +func isolateBillingSettlementBacklogState(t *testing.T) { + t.Helper() + + smartOpsAlertMonitor.Lock() + originalActiveAlerts := smartOpsAlertMonitor.active + smartOpsAlertMonitor.active = make(map[string]SmartOpsAlert) + smartOpsAlertMonitor.Unlock() + billingSettlementBacklogAlertState.Lock() originalState := struct { active bool lastCount int64 oldestCreatedAt int64 lastNotifiedAt time.Time + lastObservedAt time.Time }{ active: billingSettlementBacklogAlertState.active, lastCount: billingSettlementBacklogAlertState.lastCount, oldestCreatedAt: billingSettlementBacklogAlertState.oldestCreatedAt, lastNotifiedAt: billingSettlementBacklogAlertState.lastNotifiedAt, + lastObservedAt: billingSettlementBacklogAlertState.lastObservedAt, } billingSettlementBacklogAlertState.active = false billingSettlementBacklogAlertState.lastCount = 0 billingSettlementBacklogAlertState.oldestCreatedAt = 0 billingSettlementBacklogAlertState.lastNotifiedAt = time.Time{} + billingSettlementBacklogAlertState.lastObservedAt = time.Time{} billingSettlementBacklogAlertState.Unlock() + t.Cleanup(func() { - billingSettlementBacklogAlertNotificationSender = originalSender + smartOpsAlertMonitor.Lock() + smartOpsAlertMonitor.active = originalActiveAlerts + smartOpsAlertMonitor.Unlock() + billingSettlementBacklogAlertState.Lock() billingSettlementBacklogAlertState.active = originalState.active billingSettlementBacklogAlertState.lastCount = originalState.lastCount billingSettlementBacklogAlertState.oldestCreatedAt = originalState.oldestCreatedAt billingSettlementBacklogAlertState.lastNotifiedAt = originalState.lastNotifiedAt + billingSettlementBacklogAlertState.lastObservedAt = originalState.lastObservedAt billingSettlementBacklogAlertState.Unlock() }) +} + +func TestBillingSettlementBacklogAlertReportsCountAgeAndRecovery(t *testing.T) { + isolateBillingSettlementBacklogState(t) + originalSender := billingSettlementBacklogAlertNotificationSender + t.Cleanup(func() { + billingSettlementBacklogAlertNotificationSender = originalSender + }) var alerts []SmartOpsAlert billingSettlementBacklogAlertNotificationSender = func(alert SmartOpsAlert) { @@ -83,6 +106,10 @@ func TestBillingSettlementBacklogAlertReportsCountAgeAndRecovery(t *testing.T) { assert.Equal(t, (2 * time.Hour).Seconds(), alerts[0].Threshold) assert.Contains(t, alerts[0].Message, "2 条") assert.Contains(t, alerts[0].Message, "2h0m0s") + activeAlerts := GetSmartOpsAlerts() + require.Len(t, activeAlerts, 1) + assert.Equal(t, "billing_settlement_backlog", activeAlerts[0].Key) + assert.Equal(t, float64(2), activeAlerts[0].CurrentValue) observeBillingSettlementBacklog(stats, now.Add(time.Minute)) require.Len(t, alerts, 1, "an unchanged backlog must be deduplicated inside the reminder interval") @@ -91,6 +118,9 @@ func TestBillingSettlementBacklogAlertReportsCountAgeAndRecovery(t *testing.T) { observeBillingSettlementBacklog(stats, now.Add(2*time.Minute)) require.Len(t, alerts, 2) assert.Equal(t, float64(3), alerts[1].CurrentValue) + activeAlerts = GetSmartOpsAlerts() + require.Len(t, activeAlerts, 1) + assert.Equal(t, float64(3), activeAlerts[0].CurrentValue) observeBillingSettlementBacklog(stats, now.Add(2*time.Minute+billingSettlementBacklogNotificationInterval)) require.Len(t, alerts, 3, "a persistent backlog must produce an age reminder") @@ -99,6 +129,140 @@ func TestBillingSettlementBacklogAlertReportsCountAgeAndRecovery(t *testing.T) { require.Len(t, alerts, 4) assert.Equal(t, smartOpsAlertStatusResolved, alerts[3].Status) assert.Contains(t, alerts[3].Message, "此前共 3 条") + assert.Contains(t, alerts[3].Message, "尚未核对") + assert.Contains(t, alerts[3].Message, "pending/manual") + assert.NotContains(t, alerts[3].Message, "未完成的正向最终计费对账已清空") + assert.Empty(t, GetSmartOpsAlerts()) +} + +func TestBillingSettlementBacklogReviewRefreshDoesNotBroadcast(t *testing.T) { + isolateBillingSettlementBacklogState(t) + originalSender := billingSettlementBacklogAlertNotificationSender + billingSettlementBacklogAlertState.Lock() + now := time.Unix(40_000, 0) + billingSettlementBacklogAlertState.active = true + billingSettlementBacklogAlertState.lastCount = 48 + billingSettlementBacklogAlertState.oldestCreatedAt = now.Add(-time.Hour).Unix() + billingSettlementBacklogAlertState.lastNotifiedAt = now.Add(-time.Minute) + billingSettlementBacklogAlertState.lastObservedAt = now.Add(-time.Second) + billingSettlementBacklogAlertState.Unlock() + t.Cleanup(func() { + billingSettlementBacklogAlertNotificationSender = originalSender + }) + + var alerts []SmartOpsAlert + billingSettlementBacklogAlertNotificationSender = func(alert SmartOpsAlert) { + alerts = append(alerts, alert) + } + stats := model.BillingSettlementBacklogStats{ + Count: 47, + OldestCreatedAt: now.Add(-time.Hour).Unix(), + } + + updateBillingSettlementBacklogAlert(stats, now, false) + + assert.Empty(t, alerts, "administrator review must not broadcast a new firing notification") + activeAlerts := GetSmartOpsAlerts() + require.Len(t, activeAlerts, 1) + assert.Equal(t, float64(47), activeAlerts[0].CurrentValue) +} + +func TestBillingSettlementBacklogReviewRefreshRejectsOlderPeriodicSnapshot(t *testing.T) { + isolateBillingSettlementBacklogState(t) + originalSender := billingSettlementBacklogAlertNotificationSender + billingSettlementBacklogAlertState.Lock() + base := time.Unix(50_000, 0) + billingSettlementBacklogAlertState.active = true + billingSettlementBacklogAlertState.lastCount = 48 + billingSettlementBacklogAlertState.oldestCreatedAt = base.Add(-time.Hour).Unix() + billingSettlementBacklogAlertState.lastNotifiedAt = base.Add(-time.Minute) + billingSettlementBacklogAlertState.lastObservedAt = base.Add(-time.Minute) + billingSettlementBacklogAlertState.Unlock() + t.Cleanup(func() { + billingSettlementBacklogAlertNotificationSender = originalSender + }) + + var alerts []SmartOpsAlert + billingSettlementBacklogAlertNotificationSender = func(alert SmartOpsAlert) { + alerts = append(alerts, alert) + } + + updateBillingSettlementBacklogAlert(model.BillingSettlementBacklogStats{}, base, false) + updateBillingSettlementBacklogAlert(model.BillingSettlementBacklogStats{ + Count: 48, + OldestCreatedAt: base.Add(-time.Hour).Unix(), + }, base.Add(-time.Second), true) + + assert.Empty(t, alerts, "a stale periodic snapshot must not broadcast after an administrator refresh") + assert.Empty(t, GetSmartOpsAlerts(), "a stale periodic snapshot must not reactivate the closed alert") +} + +func TestBillingSettlementBacklogProjectionPreservesNewestObservation(t *testing.T) { + isolateBillingSettlementBacklogState(t) + originalProjector := billingSettlementBacklogAlertProjector + t.Cleanup(func() { + billingSettlementBacklogAlertProjector = originalProjector + }) + + olderObservedAt := time.Unix(60_000, 0) + newerObservedAt := olderObservedAt.Add(time.Second) + olderProjectionStarted := make(chan struct{}) + releaseOlderProjection := make(chan struct{}) + newerProjectionStarted := make(chan struct{}, 1) + var olderProjectionOnce sync.Once + billingSettlementBacklogAlertProjector = func(alert SmartOpsAlert) { + if alert.Status == smartOpsAlertStatusFiring && alert.ObservedAt.Equal(olderObservedAt) { + olderProjectionOnce.Do(func() { close(olderProjectionStarted) }) + <-releaseOlderProjection + } + if alert.Status == smartOpsAlertStatusResolved && alert.ObservedAt.Equal(newerObservedAt) { + select { + case newerProjectionStarted <- struct{}{}: + default: + } + } + projectSmartOpsActiveAlert(alert) + } + + olderDone := make(chan struct{}) + go func() { + defer close(olderDone) + updateBillingSettlementBacklogAlert(model.BillingSettlementBacklogStats{ + Count: 48, + OldestCreatedAt: olderObservedAt.Add(-time.Hour).Unix(), + }, olderObservedAt, false) + }() + select { + case <-olderProjectionStarted: + case <-time.After(time.Second): + t.Fatal("older observation did not reach the projection seam") + } + + newerDone := make(chan struct{}) + go func() { + defer close(newerDone) + updateBillingSettlementBacklogAlert(model.BillingSettlementBacklogStats{}, newerObservedAt, false) + }() + select { + case <-newerProjectionStarted: + // The unfixed implementation lets the newer observation overtake the + // older projection, which leaves the stale firing alert as the final state. + case <-time.After(100 * time.Millisecond): + // The fixed implementation serializes observation state and projection. + } + close(releaseOlderProjection) + for name, done := range map[string]<-chan struct{}{ + "older observation": olderDone, + "newer observation": newerDone, + } { + select { + case <-done: + case <-time.After(time.Second): + t.Fatalf("%s did not complete", name) + } + } + + assert.Empty(t, GetSmartOpsAlerts(), "the newest resolved observation must remain authoritative") } func (f *uncertainFundingSource) Source() string { return BillingSourceWallet } diff --git a/service/smart_ops_alert.go b/service/smart_ops_alert.go index 3f61dc80..e0d7b2f4 100644 --- a/service/smart_ops_alert.go +++ b/service/smart_ops_alert.go @@ -2,17 +2,24 @@ package service import ( "context" + "errors" "fmt" "hash/fnv" "sort" + "strconv" + "strings" "sync" "time" "github.com/MAX-API-Next/MAX-API/common" "github.com/MAX-API-Next/MAX-API/dto" "github.com/MAX-API-Next/MAX-API/model" + "github.com/MAX-API-Next/MAX-API/setting/billing_reconciliation_setting" ) +var ErrInvalidBillingSettlementReconciliationQuery = errors.New("invalid billing settlement reconciliation query") +var ErrInvalidBillingSettlementReconciliationReview = errors.New("invalid billing settlement reconciliation review") + const ( smartOpsAlertStatusFiring = "firing" smartOpsAlertStatusResolved = "resolved" @@ -30,8 +37,8 @@ var smartOpsAlertRetryDelays = []time.Duration{ } // SmartOpsAlert is the read-only incident projection exposed to administrators. -// It intentionally contains only resource metadata and never includes secrets, -// request bodies, channel keys, or billing data. +// It intentionally excludes secrets, request bodies, channel keys, and +// mutation controls. Component-specific values retain their own units. type SmartOpsAlert struct { Key string `json:"key"` Status string `json:"status"` @@ -154,7 +161,10 @@ func StartSmartOpsAlertMonitor() { smartOpsAlertMonitor.cancel = cancel smartOpsAlertMonitor.done = done smartOpsAlertMonitor.states = make(map[string]*smartOpsAlertState) - smartOpsAlertMonitor.active = make(map[string]SmartOpsAlert) + if smartOpsAlertMonitor.active == nil { + smartOpsAlertMonitor.active = make(map[string]SmartOpsAlert) + } + clearSmartOpsSystemAlertsLocked() go func() { defer close(done) runSmartOpsAlertMonitor(ctx) @@ -206,7 +216,7 @@ func evaluateSmartOpsSystemAlerts(status common.SystemStatus, config common.Perf for _, state := range smartOpsAlertMonitor.states { state.suppress(status.ObservedAt) } - clear(smartOpsAlertMonitor.active) + clearSmartOpsSystemAlertsLocked() smartOpsAlertMonitor.Unlock() return } @@ -282,6 +292,25 @@ func evaluateSmartOpsSystemAlerts(status common.SystemStatus, config common.Perf } } +func clearSmartOpsSystemAlertsLocked() { + for _, definition := range smartOpsAlertDefinitions { + delete(smartOpsAlertMonitor.active, definition.key) + } +} + +func projectSmartOpsActiveAlert(alert SmartOpsAlert) { + smartOpsAlertMonitor.Lock() + defer smartOpsAlertMonitor.Unlock() + if smartOpsAlertMonitor.active == nil { + smartOpsAlertMonitor.active = make(map[string]SmartOpsAlert) + } + if alert.Status == smartOpsAlertStatusResolved { + delete(smartOpsAlertMonitor.active, alert.Key) + return + } + smartOpsAlertMonitor.active[alert.Key] = alert +} + func formatSmartOpsAlertMessage(label string, event *smartOpsAlertEvent) string { if event.Status == smartOpsAlertStatusResolved { return fmt.Sprintf("%s 已恢复：当前 %.1f%%，阈值 %.1f%%", label, event.CurrentValue, event.Threshold) @@ -311,6 +340,59 @@ func GetSmartOpsAlerts() []SmartOpsAlert { return alerts } +// GetBillingSettlementReconciliation returns bounded, read-only settlement +// evidence for administrators. It never retries or mutates financial state. +func GetBillingSettlementReconciliation(limit int) (model.BillingSettlementReconciliationData, error) { + if limit == 0 { + limit = 100 + } + if limit < 1 || limit > 200 { + return model.BillingSettlementReconciliationData{}, fmt.Errorf( + "%w: limit must be between 1 and 200", + ErrInvalidBillingSettlementReconciliationQuery, + ) + } + return model.GetUnresolvedPositiveFinalizeSettlements(limit) +} + +func UpdateBillingSettlementBlockingPolicy(blockUserByDefault bool) error { + return model.UpdateOption( + billing_reconciliation_setting.OptionKeyBlockUserByDefault, + strconv.FormatBool(blockUserByDefault), + ) +} + +func validateBillingSettlementReviewNote(note string) error { + noteLength := len([]rune(note)) + if noteLength < 3 || noteLength > 1000 { + return fmt.Errorf( + "%w: note must contain between 3 and 1000 characters", + ErrInvalidBillingSettlementReconciliationReview, + ) + } + return nil +} + +func ReviewBillingSettlement(id int64, reviewerID int, blockUser *bool, note string) (model.BillingSettlement, error) { + if id <= 0 || reviewerID <= 0 || blockUser == nil { + return model.BillingSettlement{}, ErrInvalidBillingSettlementReconciliationReview + } + note = strings.TrimSpace(note) + if err := validateBillingSettlementReviewNote(note); err != nil { + return model.BillingSettlement{}, err + } + note = strings.TrimSpace(common.SanitizePersistedLogContent(common.MaskSensitiveInfo(note))) + if err := validateBillingSettlementReviewNote(note); err != nil { + return model.BillingSettlement{}, err + } + record, err := model.ReviewBillingSettlement(id, reviewerID, *blockUser, note) + if err != nil { + return model.BillingSettlement{}, err + } + refreshBillingSettlementBacklogAfterReview() + return record, nil +} + type smartOpsAlertRecipient struct { ID int Email string diff --git a/service/smart_ops_alert_test.go b/service/smart_ops_alert_test.go index 3e2ba9a5..77ccc860 100644 --- a/service/smart_ops_alert_test.go +++ b/service/smart_ops_alert_test.go @@ -123,6 +123,50 @@ func TestEvaluateSmartOpsSystemAlertsDoesNotResolveFromInvalidProbe(t *testing.T require.Len(t, GetSmartOpsAlerts(), 1, "an unknown sample must preserve the active incident") } +func TestDisablingSystemMonitoringPreservesBillingAlerts(t *testing.T) { + smartOpsAlertMonitor.Lock() + originalStates := smartOpsAlertMonitor.states + originalActive := smartOpsAlertMonitor.active + smartOpsAlertMonitor.states = make(map[string]*smartOpsAlertState) + smartOpsAlertMonitor.active = map[string]SmartOpsAlert{ + "billing_settlement_backlog": { + Key: "billing_settlement_backlog", + Status: smartOpsAlertStatusFiring, + Component: "billing", + }, + } + smartOpsAlertMonitor.Unlock() + t.Cleanup(func() { + smartOpsAlertMonitor.Lock() + smartOpsAlertMonitor.states = originalStates + smartOpsAlertMonitor.active = originalActive + smartOpsAlertMonitor.Unlock() + }) + + evaluateSmartOpsSystemAlerts(common.SystemStatus{}, common.PerformanceMonitorConfig{Enabled: false}, time.Now()) + + alerts := GetSmartOpsAlerts() + require.Len(t, alerts, 1) + require.Equal(t, "billing_settlement_backlog", alerts[0].Key) +} + +func TestBillingSettlementReconciliationRejectsUnboundedLimit(t *testing.T) { + _, err := GetBillingSettlementReconciliation(201) + require.ErrorIs(t, err, ErrInvalidBillingSettlementReconciliationQuery) +} + +func TestBillingSettlementReviewRequiresExplicitPolicyAndReason(t *testing.T) { + _, err := ReviewBillingSettlement(1, 2, nil, "verified") + require.ErrorIs(t, err, ErrInvalidBillingSettlementReconciliationReview) + + block := false + _, err = ReviewBillingSettlement(1, 2, &block, "x") + require.ErrorIs(t, err, ErrInvalidBillingSettlementReconciliationReview) + + _, err = ReviewBillingSettlement(1, 2, &block, "\x01\x02\x03") + require.ErrorIs(t, err, ErrInvalidBillingSettlementReconciliationReview) +} + func TestSmartOpsAlertNotifyTypeSeparatesNodes(t *testing.T) { alert := SmartOpsAlert{Key: "system_cpu", Node: "node-a", Status: smartOpsAlertStatusFiring} nodeAType := smartOpsAlertNotifyType(alert) diff --git a/setting/billing_reconciliation_setting/config.go b/setting/billing_reconciliation_setting/config.go new file mode 100644 index 00000000..b2d79d50 --- /dev/null +++ b/setting/billing_reconciliation_setting/config.go @@ -0,0 +1,39 @@ +package billing_reconciliation_setting + +import ( + "strconv" + + "github.com/MAX-API-Next/MAX-API/common" + "github.com/MAX-API-Next/MAX-API/setting/config" +) + +const OptionKeyBlockUserByDefault = "billing_reconciliation_setting.block_user_by_default" + +type BillingReconciliationSetting struct { + BlockUserByDefault bool `json:"block_user_by_default"` +} + +var billingReconciliationSetting = BillingReconciliationSetting{ + BlockUserByDefault: true, +} + +func init() { + config.GlobalConfig.Register("billing_reconciliation_setting", &billingReconciliationSetting) +} + +// BlockUserByDefault returns the persisted admission policy. Reading through +// OptionMap keeps the hot path synchronized with runtime option updates while +// retaining a fail-closed default before options are initialized. +func BlockUserByDefault() bool { + common.OptionMapRWMutex.RLock() + raw, ok := common.OptionMap[OptionKeyBlockUserByDefault] + common.OptionMapRWMutex.RUnlock() + if !ok { + return true + } + enabled, err := strconv.ParseBool(raw) + if err != nil { + return true + } + return enabled +} diff --git a/setting/billing_reconciliation_setting/config_test.go b/setting/billing_reconciliation_setting/config_test.go new file mode 100644 index 00000000..374d79ef --- /dev/null +++ b/setting/billing_reconciliation_setting/config_test.go @@ -0,0 +1,45 @@ +package billing_reconciliation_setting + +import ( + "testing" + + "github.com/MAX-API-Next/MAX-API/common" + "github.com/MAX-API-Next/MAX-API/setting/config" + "github.com/stretchr/testify/assert" +) + +func TestDefaultBlockingPolicyIsRegisteredFailClosed(t *testing.T) { + registered := config.GlobalConfig.ExportAllConfigs() + + assert.Equal(t, "true", registered[OptionKeyBlockUserByDefault]) + + restoreOptionMap := replaceOptionMapForTest(map[string]string{}) + t.Cleanup(restoreOptionMap) + assert.True(t, BlockUserByDefault()) +} + +func TestBlockUserByDefaultParsesRuntimeOptionFailClosed(t *testing.T) { + restoreOptionMap := replaceOptionMapForTest(map[string]string{ + OptionKeyBlockUserByDefault: "false", + }) + t.Cleanup(restoreOptionMap) + + assert.False(t, BlockUserByDefault()) + + common.OptionMapRWMutex.Lock() + common.OptionMap[OptionKeyBlockUserByDefault] = "not-a-boolean" + common.OptionMapRWMutex.Unlock() + assert.True(t, BlockUserByDefault()) +} + +func replaceOptionMapForTest(replacement map[string]string) func() { + common.OptionMapRWMutex.Lock() + original := common.OptionMap + common.OptionMap = replacement + common.OptionMapRWMutex.Unlock() + return func() { + common.OptionMapRWMutex.Lock() + common.OptionMap = original + common.OptionMapRWMutex.Unlock() + } +} diff --git a/web/default/src/features/smart-ops/alerts.test.tsx b/web/default/src/features/smart-ops/alerts.test.tsx index d85570b8..2b242084 100644 --- a/web/default/src/features/smart-ops/alerts.test.tsx +++ b/web/default/src/features/smart-ops/alerts.test.tsx @@ -20,9 +20,10 @@ import { QueryClient, QueryClientProvider } from '@tanstack/react-query' import { createReactTestEnvironment } from '@/test/react' import { waitFor, within } from '@testing-library/react' import assert from 'node:assert/strict' -import { after, before, describe, test } from 'node:test' +import { after, describe, test } from 'node:test' +import { useAuthStore } from '@/stores/auth-store' import { api } from '@/lib/api' -import { ActiveAlerts } from './alerts' +import type { BillingSettlementReconciliationData } from './types' const LOAD_ERROR_KEY = 'We could not load active alerts.' const testEnv = createReactTestEnvironment({ @@ -36,7 +37,8 @@ const testEnv = createReactTestEnvironment({ }, }) -before(() => testEnv.setup()) +await testEnv.setup() +const { ActiveAlerts } = await import('./alerts') after(() => testEnv.teardown()) @@ -49,12 +51,32 @@ function createQueryClient(): QueryClient { }) } +function emptyReconciliationData(): BillingSettlementReconciliationData { + return { + total_count: 0, + pending_count: 0, + manual_count: 0, + open_alert_count: 0, + reviewed_count: 0, + blocking_record_count: 0, + blocked_user_count: 0, + block_user_by_default: true, + oldest_created_at: 0, + truncated: false, + generated_at: 1788106455, + items: [], + } +} + describe('SmartOps active alerts', () => { - test('polls the administrator alert endpoint and renders active host pressure', async () => { + test('polls the administrator alert endpoint and renders active host pressure', async (): Promise => { const originalGet = api.get const urls: string[] = [] - api.get = (async (url) => { + api.get = (async (url: string): Promise => { urls.push(String(url)) + if (url === '/api/smart-ops/billing-settlements') { + return { data: { success: true, data: emptyReconciliationData() } } + } return { data: { success: true, @@ -99,26 +121,29 @@ describe('SmartOps active alerts', () => { } }) - test('formats percentages with the active locale', async () => { + test('formats percentages with the active locale', async (): Promise => { const originalGet = api.get await testEnv.i18n.changeLanguage('fr') - api.get = (async () => ({ - data: { - success: true, - data: [ - { - key: 'system_memory', - status: 'firing', - severity: 'warning', - component: 'system', - node: 'node-fr', - current_value: 95.25, - threshold: 90, - observed_at: '2026-08-22T08:00:00Z', - message: 'Memory usage exceeded the threshold', - }, - ], - }, + api.get = (async (url: string): Promise => ({ + data: + url === '/api/smart-ops/billing-settlements' + ? { success: true, data: emptyReconciliationData() } + : { + success: true, + data: [ + { + key: 'system_memory', + status: 'firing', + severity: 'warning', + component: 'system', + node: 'node-fr', + current_value: 95.25, + threshold: 90, + observed_at: '2026-08-22T08:00:00Z', + message: 'Memory usage exceeded the threshold', + }, + ], + }, })) as typeof api.get const queryClient = createQueryClient() @@ -130,7 +155,7 @@ describe('SmartOps active alerts', () => { try { await waitFor(() => { - assert.ok((view.container.textContent ?? '').includes('95,3%')) + assert.ok((view.container.textContent ?? '').includes('95,3\u00a0%')) }) } finally { api.get = originalGet @@ -140,10 +165,442 @@ describe('SmartOps active alerts', () => { } }) - test('shows a healthy empty state when no incident is active', async () => { + test('renders billing backlog values and read-only reconciliation evidence', async (): Promise => { const originalGet = api.get - api.get = (async () => ({ - data: { success: true, data: [] }, + const urls: string[] = [] + api.get = (async (url: string): Promise => { + urls.push(String(url)) + if (url === '/api/smart-ops/billing-settlements') { + return { + data: { + success: true, + data: { + total_count: 48, + pending_count: 17, + manual_count: 31, + open_alert_count: 1, + reviewed_count: 47, + blocking_record_count: 1, + blocked_user_count: 9, + block_user_by_default: true, + oldest_created_at: 1786032544, + truncated: false, + generated_at: 1788106455, + items: [ + { + id: 71, + operation_key: 'request:billing-request-71:finalize', + status: 'manual', + source: 'wallet', + user_id: 42, + subscription_id: 0, + token_id: 84, + task_id: 0, + funding_delta: 2500, + applied_funding_delta: 0, + token_delta: 2500, + applied_token_delta: 0, + attempts: 1, + last_error: 'user quota is not enough', + next_attempt: 0, + created_at: 1786032544, + updated_at: 1786032544, + reconciliation_reviewed_at: 0, + reconciliation_reviewed_by: 0, + reconciliation_review_note: '', + user_blocking_override: false, + record_blocks_user: false, + blocks_user: true, + }, + ], + }, + }, + } + } + return { + data: { + success: true, + data: [ + { + key: 'billing_settlement_backlog', + status: 'firing', + severity: 'warning', + component: 'billing', + node: 'XG', + current_value: 48, + threshold: 2121911, + observed_at: '2026-08-30T16:14:15Z', + message: 'billing backlog', + }, + ], + }, + } + }) as typeof api.get + + const queryClient = createQueryClient() + const view = await testEnv.render( + + + + ) + + try { + await waitFor(() => { + const text = view.container.textContent ?? '' + assert.ok(urls.includes('/api/smart-ops/alerts')) + assert.ok(urls.includes('/api/smart-ops/billing-settlements')) + assert.ok(text.includes('Billing reconciliation backlog')) + assert.ok(text.includes('48 records')) + assert.ok(!text.includes('4,800.0%')) + assert.ok(text.includes('Pending: 17')) + assert.ok(text.includes('Manual: 31')) + assert.ok(text.includes('Blocked users: 9')) + assert.ok(text.includes('request:billing-request-71:finalize')) + assert.ok(text.includes('user quota is not enough')) + assert.ok(text.includes('User blocked')) + assert.ok(text.includes('Record policy: allow')) + }) + const reviewButton = within(view.container).getByRole('button', { + name: 'Review and close', + }) + await view.click(reviewButton) + let dialog: HTMLElement | undefined + await waitFor(() => { + dialog = within(document.body).getByRole('dialog', { + name: 'Review billing reconciliation', + }) + }) + assert.ok(dialog) + const noteInput = within(dialog).getByRole('textbox', { + name: 'Review note', + }) + assert.equal(noteInput.getAttribute('aria-invalid'), 'true') + assert.ok( + (dialog.textContent ?? '').includes( + 'Review note must contain between 3 and 1000 characters.' + ) + ) + assert.equal( + within(dialog) + .getByRole('button', { name: 'Close alert' }) + .hasAttribute('disabled'), + true + ) + assert.equal( + within(dialog) + .getByRole('button', { name: 'Allow user to continue' }) + .getAttribute('aria-pressed'), + 'true' + ) + await view.click(within(dialog).getByRole('button', { name: 'Cancel' })) + } finally { + api.get = originalGet + queryClient.clear() + await view.unmount() + } + }) + + test('keeps the billing alert visible when reconciliation details fail', async (): Promise => { + const originalGet = api.get + api.get = (async (url: string): Promise => { + if (url === '/api/smart-ops/billing-settlements') { + throw new Error('temporary reconciliation projection failure') + } + return { + data: { + success: true, + data: [ + { + key: 'billing_settlement_backlog', + status: 'firing', + severity: 'warning', + component: 'billing', + node: 'XG', + current_value: 48, + threshold: 2121911, + observed_at: '2026-08-30T16:14:15Z', + message: 'billing backlog', + }, + ], + }, + } + }) as typeof api.get + + const queryClient = createQueryClient() + const view = await testEnv.render( + + + + ) + + try { + await waitFor(() => { + const text = view.container.textContent ?? '' + assert.ok(text.includes('Billing reconciliation backlog')) + assert.ok(text.includes('48 records')) + assert.ok( + text.includes('We could not load billing reconciliation details.') + ) + assert.ok(text.includes('temporary reconciliation projection failure')) + }) + } finally { + api.get = originalGet + queryClient.clear() + await view.unmount() + } + }) + + test('routes malformed reconciliation payloads to the existing error state', async (): Promise => { + const originalGet = api.get + api.get = (async (url: string): Promise => ({ + data: + url === '/api/smart-ops/billing-settlements' + ? { + success: true, + data: { ...emptyReconciliationData(), items: [null] }, + } + : { success: true, data: [] }, + })) as typeof api.get + + const queryClient = createQueryClient() + const view = await testEnv.render( + + + + ) + + try { + await waitFor(() => { + const text = view.container.textContent ?? '' + assert.ok( + text.includes('We could not load billing reconciliation details.') + ) + assert.ok(!text.includes('No unresolved reconciliation records.')) + }) + } finally { + api.get = originalGet + queryClient.clear() + await view.unmount() + } + }) + + test('loads reviewed reconciliation without an active alert and submits administrator controls', async (): Promise => { + const originalUser = useAuthStore.getState().auth.user + useAuthStore.getState().auth.setUser({ + id: 1, + username: 'root', + role: 100, + }) + const originalGet = api.get + const originalPut = api.put + const originalPost = api.post + const writes: Array<{ method: string; url: string; data: unknown }> = [] + let reconciliationRequests = 0 + let blockUserByDefault = true + api.get = (async (url: string): Promise => { + if (url === '/api/smart-ops/billing-settlements') { + reconciliationRequests += 1 + return { + data: { + success: true, + data: { + ...emptyReconciliationData(), + total_count: 1, + pending_count: 1, + open_alert_count: 0, + reviewed_count: 2, + blocking_record_count: 0, + blocked_user_count: 0, + block_user_by_default: blockUserByDefault, + items: [ + { + id: 91, + operation_key: 'request:billing-request-91:finalize', + status: 'pending', + source: 'wallet', + user_id: 51, + subscription_id: 0, + token_id: 0, + task_id: 0, + funding_delta: 100, + applied_funding_delta: 0, + token_delta: 100, + applied_token_delta: 0, + attempts: 2, + last_error: 'quota changed', + next_attempt: 1788106500, + created_at: 1786032544, + updated_at: 1786032544, + reconciliation_reviewed_at: 1788106400, + reconciliation_reviewed_by: 7, + reconciliation_review_note: '🙂🙂🙂', + user_blocking_override: false, + record_blocks_user: false, + blocks_user: false, + }, + ], + }, + }, + } + } + return { data: { success: true, data: [] } } + }) as typeof api.get + api.put = (async (url: string, data: unknown): Promise => { + writes.push({ method: 'PUT', url: String(url), data }) + blockUserByDefault = (data as { block_user_by_default: boolean }) + .block_user_by_default + return { data: { success: true } } + }) as typeof api.put + api.post = (async (url: string, data: unknown): Promise => { + writes.push({ method: 'POST', url: String(url), data }) + return { data: { success: true } } + }) as typeof api.post + + const queryClient = createQueryClient() + const view = await testEnv.render( + + + + ) + + try { + let policySwitch: HTMLElement | undefined + let reviewButton: HTMLElement | undefined + await waitFor(() => { + const screen = within(view.container) + assert.ok( + (view.container.textContent ?? '').includes('No active alerts.') + ) + assert.ok( + (view.container.textContent ?? '').includes( + 'request:billing-request-91:finalize' + ) + ) + assert.ok( + (view.container.textContent ?? '').includes('Reviewed records: 2') + ) + policySwitch = screen.getByRole('switch', { + name: 'Block affected users by default', + }) + reviewButton = screen.getByRole('button', { + name: 'Edit review', + }) + }) + + assert.ok(policySwitch) + await view.click(policySwitch) + await waitFor(() => { + assert.deepEqual(writes[0], { + method: 'PUT', + url: '/api/smart-ops/billing-settlements/blocking-policy', + data: { block_user_by_default: false }, + }) + assert.ok(reconciliationRequests > 1) + }) + + reviewButton = within(view.container).getByRole('button', { + name: 'Edit review', + }) + await view.click(reviewButton) + let dialog: HTMLElement | undefined + await waitFor(() => { + dialog = within(document.body).getByRole('dialog') + }) + assert.ok(dialog) + const noteInput = within(dialog).getByRole('textbox', { + name: 'Review note', + }) as HTMLTextAreaElement + assert.equal(noteInput.value, '🙂🙂🙂') + assert.ok((dialog.textContent ?? '').includes('3 / 1000 characters')) + const closeAlertButton = within(dialog).getByRole('button', { + name: 'Save review', + }) + await waitFor(() => + assert.equal(closeAlertButton.hasAttribute('disabled'), false) + ) + const reconciliationRequestsBeforeReview = reconciliationRequests + await view.click(closeAlertButton) + + await waitFor(() => { + assert.deepEqual(writes[1], { + method: 'POST', + url: '/api/smart-ops/billing-settlements/91/review', + data: { + block_user: false, + note: '🙂🙂🙂', + }, + }) + assert.ok(reconciliationRequests > reconciliationRequestsBeforeReview) + }) + } finally { + api.get = originalGet + api.put = originalPut + api.post = originalPost + await view.unmount() + queryClient.clear() + useAuthStore.getState().auth.setUser(originalUser) + } + }) + + test('keeps the global blocking policy read only for non-root administrators', async (): Promise => { + const originalUser = useAuthStore.getState().auth.user + useAuthStore.getState().auth.setUser({ + id: 2, + username: 'admin', + role: 10, + }) + const originalGet = api.get + const originalPut = api.put + let policyWrites = 0 + api.get = (async (url: string): Promise => ({ + data: + url === '/api/smart-ops/billing-settlements' + ? { success: true, data: emptyReconciliationData() } + : { success: true, data: [] }, + })) as typeof api.get + api.put = (async (): Promise => { + policyWrites += 1 + return { data: { success: true } } + }) as typeof api.put + + const queryClient = createQueryClient() + const view = await testEnv.render( + + + + ) + + try { + let policySwitch: HTMLElement | undefined + await waitFor(() => { + policySwitch = within(view.container).getByRole('switch', { + name: 'Block affected users by default', + }) + assert.ok( + (view.container.textContent ?? '').includes( + 'Only root administrators can change the default blocking policy.' + ) + ) + }) + assert.ok(policySwitch) + assert.equal(policySwitch.hasAttribute('data-disabled'), true) + assert.equal(policyWrites, 0) + } finally { + api.get = originalGet + api.put = originalPut + await view.unmount() + queryClient.clear() + useAuthStore.getState().auth.setUser(originalUser) + } + }) + + test('shows a healthy empty state when no incident is active', async (): Promise => { + const originalGet = api.get + api.get = (async (url: string): Promise => ({ + data: + url === '/api/smart-ops/billing-settlements' + ? { success: true, data: emptyReconciliationData() } + : { success: true, data: [] }, })) as typeof api.get const queryClient = createQueryClient() @@ -166,11 +623,14 @@ describe('SmartOps active alerts', () => { } }) - test('shows a localized fallback for an unsuccessful response', async () => { + test('shows a localized fallback for an unsuccessful response', async (): Promise => { const originalGet = api.get await testEnv.i18n.changeLanguage('fr') - api.get = (async () => ({ - data: { success: false, data: [] }, + api.get = (async (url: string): Promise => ({ + data: + url === '/api/smart-ops/billing-settlements' + ? { success: true, data: emptyReconciliationData() } + : { success: false, data: [] }, })) as typeof api.get const queryClient = createQueryClient() @@ -194,10 +654,13 @@ describe('SmartOps active alerts', () => { } }) - test('shows a retryable error state when the alert endpoint fails', async () => { + test('shows a retryable error state when the alert endpoint fails', async (): Promise => { const originalGet = api.get let requestCount = 0 - api.get = (async () => { + api.get = (async (url: string): Promise => { + if (url === '/api/smart-ops/billing-settlements') { + return { data: { success: true, data: emptyReconciliationData() } } + } requestCount += 1 throw new Error('temporary alert endpoint failure') }) as typeof api.get diff --git a/web/default/src/features/smart-ops/alerts.tsx b/web/default/src/features/smart-ops/alerts.tsx index 9f53e8b0..d92b669d 100644 --- a/web/default/src/features/smart-ops/alerts.tsx +++ b/web/default/src/features/smart-ops/alerts.tsx @@ -20,7 +20,9 @@ import type { ReactElement } from 'react' import { useQuery } from '@tanstack/react-query' import { RefreshCw, ShieldCheck, TriangleAlert } from 'lucide-react' import { useTranslation } from 'react-i18next' +import { useAuthStore } from '@/stores/auth-store' import { formatTimestampRelative, formatTimestampToDate } from '@/lib/format' +import { ROLE } from '@/lib/roles' import { cn } from '@/lib/utils' import { Badge } from '@/components/ui/badge' import { Button } from '@/components/ui/button' @@ -42,27 +44,30 @@ import { } from '@/components/ui/table' import { ErrorState } from '@/components/error-state' import { SectionPageLayout } from '@/components/layout' -import { getSmartOpsAlerts } from './api' -import type { SmartOpsAlert } from './types' +import { getBillingSettlementReconciliation, getSmartOpsAlerts } from './api' +import { BillingSettlementEvidence } from './components/billing-settlement-evidence' +import { + formatAlertCurrentValue, + formatAlertThreshold, + getObservedAtMilliseconds, +} from './lib/format' +import { + SMART_OPS_ACTIVE_ALERTS_QUERY_KEY, + SMART_OPS_BILLING_RECONCILIATION_QUERY_KEY, +} from './lib/query-keys' +import { isBillingSettlementReconciliationData } from './lib/reconciliation-validation' +import type { + BillingSettlementReconciliationData, + SmartOpsAlert, +} from './types' -const ALERT_POLL_INTERVAL_MS = 5000 +const ALERT_POLL_INTERVAL_MS = 30000 const RESOURCE_LABEL: Record = { system_cpu: 'CPU usage', system_memory: 'Memory usage', system_disk: 'Disk usage', -} - -function formatPercent(value: number, locale: string): string { - return `${new Intl.NumberFormat(locale, { - minimumFractionDigits: 1, - maximumFractionDigits: 1, - }).format(value)}%` -} - -function getObservedAtMilliseconds(alert: SmartOpsAlert): number | undefined { - const timestamp = Date.parse(alert.observed_at) - return Number.isNaN(timestamp) ? undefined : timestamp + billing_settlement_backlog: 'Billing reconciliation backlog', } interface ActiveAlertsTableProps { @@ -74,7 +79,7 @@ function ActiveAlertsTable(props: ActiveAlertsTableProps): ReactElement { return (
- +
{t('Resource')} @@ -84,7 +89,7 @@ function ActiveAlertsTable(props: ActiveAlertsTableProps): ReactElement { {t('Current Value')} - {t('Threshold')} + {t('Threshold / Oldest wait')} {t('Observed')} @@ -96,8 +101,8 @@ function ActiveAlertsTable(props: ActiveAlertsTableProps): ReactElement { const observedAt = getObservedAtMilliseconds(alert) return ( - -
+ +
{t(RESOURCE_LABEL[alert.key] ?? alert.key)} @@ -113,10 +118,10 @@ function ActiveAlertsTable(props: ActiveAlertsTableProps): ReactElement { {alert.node || t('Unknown')} - {formatPercent(alert.current_value, i18n.language)} + {formatAlertCurrentValue(alert, i18n.language, t)} - {formatPercent(alert.threshold, i18n.language)} + {formatAlertThreshold(alert, i18n.language, t)} void +} + +function ActiveAlertsContent(props: ActiveAlertsContentProps): ReactElement { + const { t } = useTranslation() + + if (props.loading) { + return ( +
+ {Array.from({ length: 3 }).map((_, index) => ( + + ))} +
+ ) + } + + if (props.isError) { + return ( + + ) + } + + if (props.alerts.length === 0) { + return ( + + + + + {t('No active alerts.')} + + {t( + 'This process is not currently reporting sustained host pressure or an open billing reconciliation alert.' + )} + + + + ) + } + + return +} + export function ActiveAlerts(): ReactElement { const { t } = useTranslation() + const userRole = useAuthStore((state) => state.auth.user?.role) + const canUpdateBlockingPolicy = userRole === ROLE.SUPER_ADMIN const loadErrorMessage = t('We could not load active alerts.') + const reconciliationErrorMessage = t( + 'We could not load billing reconciliation details.' + ) const alertsQuery = useQuery({ - queryKey: ['smart-ops', 'active-alerts', loadErrorMessage], + queryKey: [...SMART_OPS_ACTIVE_ALERTS_QUERY_KEY, loadErrorMessage], queryFn: async (): Promise => { const response = await getSmartOpsAlerts() if (!response.success || !Array.isArray(response.data)) { @@ -154,8 +220,28 @@ export function ActiveAlerts(): ReactElement { }) const alerts = alertsQuery.data ?? [] + const reconciliationQuery = useQuery({ + queryKey: [ + ...SMART_OPS_BILLING_RECONCILIATION_QUERY_KEY, + reconciliationErrorMessage, + ], + queryFn: async (): Promise => { + const response = await getBillingSettlementReconciliation() + if ( + !response.success || + !isBillingSettlementReconciliationData(response.data) + ) { + throw new Error(response.message || reconciliationErrorMessage) + } + return response.data + }, + retry: false, + refetchInterval: ALERT_POLL_INTERVAL_MS, + }) + const loading = alertsQuery.isLoading - const refreshing = alertsQuery.isFetching && !loading + const refreshing = + (alertsQuery.isFetching && !loading) || reconciliationQuery.isFetching return ( @@ -163,7 +249,7 @@ export function ActiveAlerts(): ReactElement { {t('Active Alerts')} - {t('Read only')} + {t('Administrator controls')} @@ -187,7 +273,7 @@ export function ActiveAlerts(): ReactElement {

{t( - 'SmartOps reports sustained host pressure here for manual investigation and repair.' + 'SmartOps reports sustained host pressure and billing reconciliation backlogs here for manual investigation.' )}

@@ -196,8 +282,13 @@ export function ActiveAlerts(): ReactElement { type='button' variant='outline' size='sm' - onClick={() => void alertsQuery.refetch()} - disabled={alertsQuery.isFetching} + onClick={() => { + void alertsQuery.refetch() + void reconciliationQuery.refetch() + }} + disabled={ + alertsQuery.isFetching || reconciliationQuery.isFetching + } > - {loading ? ( -
- {Array.from({ length: 3 }).map((_, index) => ( - - ))} -
- ) : alertsQuery.isError ? ( - + void alertsQuery.refetch()} - className='min-h-[240px]' /> - ) : alerts.length === 0 ? ( - - - - - {t('No active alerts.')} - - {t( - 'This process is not currently reporting a sustained CPU, memory, or disk alert.' - )} - - - - ) : ( -
- -
- )} + void reconciliationQuery.refetch()} + /> + diff --git a/web/default/src/features/smart-ops/api.ts b/web/default/src/features/smart-ops/api.ts index 2da83b80..5083f117 100644 --- a/web/default/src/features/smart-ops/api.ts +++ b/web/default/src/features/smart-ops/api.ts @@ -19,6 +19,9 @@ For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API import { api } from '@/lib/api' import { MAX_PERFORMANCE_LIMIT } from './lib/filters' import type { + BillingSettlementReconciliationResponse, + BillingSettlementMutationResponse, + BillingSettlementReviewRequest, ChannelPerformanceData, ChannelPerformanceQuery, ChannelPerformanceResponse, @@ -37,6 +40,36 @@ export async function getSmartOpsAlerts(): Promise { return response.data } +export async function getBillingSettlementReconciliation(): Promise { + const response = await api.get( + '/api/smart-ops/billing-settlements' + ) + return response.data +} + +export async function updateBillingSettlementBlockingPolicy( + blockUserByDefault: boolean +): Promise { + const response = await api.put( + '/api/smart-ops/billing-settlements/blocking-policy', + { block_user_by_default: blockUserByDefault }, + { skipBusinessError: true, skipErrorHandler: true } + ) + return response.data +} + +export async function reviewBillingSettlement( + id: number, + request: BillingSettlementReviewRequest +): Promise { + const response = await api.post( + `/api/smart-ops/billing-settlements/${id}/review`, + request, + { skipBusinessError: true, skipErrorHandler: true } + ) + return response.data +} + export async function getChannelPerformance( query: ChannelPerformanceQuery ): Promise { diff --git a/web/default/src/features/smart-ops/components/billing-settlement-evidence.tsx b/web/default/src/features/smart-ops/components/billing-settlement-evidence.tsx new file mode 100644 index 00000000..3089d7d6 --- /dev/null +++ b/web/default/src/features/smart-ops/components/billing-settlement-evidence.tsx @@ -0,0 +1,277 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import { useState, type ReactElement } from 'react' +import { useMutation, useQueryClient } from '@tanstack/react-query' +import { CheckCircle2, TriangleAlert } from 'lucide-react' +import { useTranslation } from 'react-i18next' +import { toast } from 'sonner' +import { formatTimestampToDate } from '@/lib/format' +import { handleServerError } from '@/lib/handle-server-error' +import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert' +import { Badge } from '@/components/ui/badge' +import { Button } from '@/components/ui/button' +import { + Field, + FieldContent, + FieldDescription, + FieldTitle, +} from '@/components/ui/field' +import { Skeleton } from '@/components/ui/skeleton' +import { Switch } from '@/components/ui/switch' +import { updateBillingSettlementBlockingPolicy } from '../api' +import { formatCount, formatLocalizedCount } from '../lib/format' +import { mutationErrorMessage } from '../lib/mutation-error' +import { + SMART_OPS_ACTIVE_ALERTS_QUERY_KEY, + SMART_OPS_BILLING_RECONCILIATION_QUERY_KEY, +} from '../lib/query-keys' +import type { + BillingSettlementReconciliationData, + BillingSettlementReconciliationItem, +} from '../types' +import { BillingSettlementTable } from './billing-settlement-table' +import { SettlementReviewDialog } from './settlement-review-dialog' + +interface BillingSettlementEvidenceProps { + canUpdateBlockingPolicy: boolean + data?: BillingSettlementReconciliationData + error: Error | null + loading: boolean + onRetry: () => void +} + +export function BillingSettlementEvidence( + props: BillingSettlementEvidenceProps +): ReactElement { + const { t, i18n } = useTranslation() + const [selectedItem, setSelectedItem] = + useState(null) + const [policyOverride, setPolicyOverride] = useState(null) + const queryClient = useQueryClient() + const policyValue = + policyOverride ?? props.data?.block_user_by_default ?? true + + const policyMutation = useMutation({ + mutationKey: ['smart-ops', 'billing-settlement-blocking-policy'], + mutationFn: async (checked: boolean) => { + const response = await updateBillingSettlementBlockingPolicy(checked) + if (!response.success) { + throw new Error( + response.message || t('Failed to update blocking policy.') + ) + } + }, + onSuccess: async () => { + toast.success(t('Default user-blocking policy updated.')) + await Promise.all([ + queryClient.invalidateQueries({ + queryKey: SMART_OPS_ACTIVE_ALERTS_QUERY_KEY, + }), + queryClient.invalidateQueries({ + queryKey: SMART_OPS_BILLING_RECONCILIATION_QUERY_KEY, + }), + ]) + }, + onError: (error) => { + handleServerError(error, { + fallback: mutationErrorMessage( + error, + t('Failed to update blocking policy.') + ), + }) + }, + onSettled: () => { + setPolicyOverride(null) + }, + }) + + const handlePolicyChange = (checked: boolean) => { + setPolicyOverride(checked) + policyMutation.mutate(checked) + } + + if (props.loading) { + return ( +
+ + +
+ ) + } + + if (props.error) { + return ( + + + ) + } + + if (!props.data) { + return <> + } + + return ( +
+
+
+

+ {t('Billing reconciliation controls')} +

+

+ {t( + 'Review and close operational alerts without changing the underlying pending or manual financial settlement.' + )} +

+
+
+ + {formatLocalizedCount( + props.data.open_alert_count, + i18n.language, + t, + 'Open alert: {{count}}', + 'Open alerts: {{count}}' + )} + + + {formatLocalizedCount( + props.data.reviewed_count, + i18n.language, + t, + 'Reviewed record: {{count}}', + 'Reviewed records: {{count}}' + )} + + + {t('Pending: {{count}}', { + count: formatCount(props.data.pending_count, i18n.language), + })} + + + {t('Manual: {{count}}', { + count: formatCount(props.data.manual_count, i18n.language), + })} + + + {formatLocalizedCount( + props.data.blocked_user_count, + i18n.language, + t, + 'Blocked user: {{count}}', + 'Blocked users: {{count}}' + )} + +
+
+ + + + {t('Block affected users by default')} + + {t( + 'When enabled, new paid requests remain blocked while any unresolved positive final settlement record still blocks the user. Allowing one reviewed record does not override other blocking records.' + )} + + {!props.canUpdateBlockingPolicy && ( + + {t( + 'Only root administrators can change the default blocking policy.' + )} + + )} + + + + + {props.data.items.length === 0 ? ( + + + ) : ( + + )} + + {props.data.truncated && ( + + + {t( + 'Showing the oldest {{count}} records; the summary covers all {{total}} unresolved records.', + { + count: formatCount(props.data.items.length, i18n.language), + total: formatCount(props.data.total_count, i18n.language), + } + )} + + + )} +

+ {t('Generated at {{time}}', { + time: formatTimestampToDate(props.data.generated_at), + })} +

+ {selectedItem && ( + { + if (!open) setSelectedItem(null) + }} + /> + )} +
+ ) +} diff --git a/web/default/src/features/smart-ops/components/billing-settlement-table.tsx b/web/default/src/features/smart-ops/components/billing-settlement-table.tsx new file mode 100644 index 00000000..60014b8d --- /dev/null +++ b/web/default/src/features/smart-ops/components/billing-settlement-table.tsx @@ -0,0 +1,252 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import type { ReactElement } from 'react' +import type { TFunction } from 'i18next' +import { Pencil } from 'lucide-react' +import { useTranslation } from 'react-i18next' +import { + formatQuota, + formatTimestampRelative, + formatTimestampToDate, +} from '@/lib/format' +import { Badge } from '@/components/ui/badge' +import { Button } from '@/components/ui/button' +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@/components/ui/table' +import { formatLocalizedCount } from '../lib/format' +import type { BillingSettlementReconciliationItem } from '../types' + +const FUNDING_SOURCE_LABEL: Record = { + wallet: 'Wallet', + subscription: 'Subscription', +} + +function settlementReferences( + item: BillingSettlementReconciliationItem, + t: TFunction +): string[] { + const references = [t('User #{{id}}', { id: item.user_id })] + if (item.subscription_id > 0) { + references.push(t('Subscription #{{id}}', { id: item.subscription_id })) + } + if (item.token_id > 0) { + references.push(t('Token #{{id}}', { id: item.token_id })) + } + if (item.task_id > 0) { + references.push(t('Task #{{id}}', { id: item.task_id })) + } + return references +} + +interface BillingSettlementTableProps { + items: BillingSettlementReconciliationItem[] + onSelectItem: (item: BillingSettlementReconciliationItem) => void +} + +export function BillingSettlementTable( + props: BillingSettlementTableProps +): ReactElement { + const { t, i18n } = useTranslation() + + return ( +
+
+ + + {t('Operation')} + {t('Financial / alert status')} + {t('User access')} + {t('References')} + {t('Funding source')} + + {t('Outstanding funding')} + + {t('Attempts / next retry')} + {t('Last error')} + {t('Administrator review')} + {t('Created')} + {t('Actions')} + + + + {props.items.map((item) => { + const outstandingFunding = + item.funding_delta - item.applied_funding_delta + const references = settlementReferences(item, t) + return ( + + + + {item.operation_key} + + + +
+ + {item.status === 'manual' ? t('Manual') : t('Pending')} + + 0 + ? 'secondary' + : 'destructive' + } + > + {item.reconciliation_reviewed_at > 0 + ? t('Reviewed') + : t('Open alert')} + +
+ + +
+ + {item.blocks_user ? t('User blocked') : t('User allowed')} + + + {item.record_blocks_user + ? t('Record policy: block') + : t('Record policy: allow')} + +
+ + +
+ {references.map((reference) => ( + {reference} + ))} +
+ + + {t(FUNDING_SOURCE_LABEL[item.source] ?? item.source)} + + + {formatQuota(outstandingFunding)} + + +
+ + {formatLocalizedCount( + item.attempts, + i18n.language, + t, + '{{count}} attempt', + '{{count}} attempts' + )} + + + {item.status === 'manual' + ? t('Manual review') + : formatTimestampRelative( + item.next_attempt, + 'seconds', + i18n.language + )} + +
+ + + + {item.last_error || t('No error detail')} + + + + {item.reconciliation_reviewed_at > 0 ? ( +
+ + {t('Administrator #{{id}}', { + id: item.reconciliation_reviewed_by, + })} + + + {formatTimestampRelative( + item.reconciliation_reviewed_at, + 'seconds', + i18n.language + )} + + + {item.reconciliation_review_note} + +
+ ) : ( + + {t('Not reviewed')} + + )} + + + {formatTimestampRelative( + item.created_at, + 'seconds', + i18n.language + )} + + + + + + ) + })} + +
+
+ ) +} diff --git a/web/default/src/features/smart-ops/components/settlement-review-dialog.tsx b/web/default/src/features/smart-ops/components/settlement-review-dialog.tsx new file mode 100644 index 00000000..7a1207dd --- /dev/null +++ b/web/default/src/features/smart-ops/components/settlement-review-dialog.tsx @@ -0,0 +1,229 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ +import { useState, type ReactElement } from 'react' +import { useMutation, useQueryClient } from '@tanstack/react-query' +import { Loader2, LockKeyhole, LockOpen } from 'lucide-react' +import { useTranslation } from 'react-i18next' +import { toast } from 'sonner' +import { handleServerError } from '@/lib/handle-server-error' +import { Button } from '@/components/ui/button' +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from '@/components/ui/dialog' +import { + Field, + FieldDescription, + FieldError, + FieldGroup, + FieldLabel, +} from '@/components/ui/field' +import { Textarea } from '@/components/ui/textarea' +import { ToggleGroup, ToggleGroupItem } from '@/components/ui/toggle-group' +import { reviewBillingSettlement } from '../api' +import { formatLocalizedCount } from '../lib/format' +import { mutationErrorMessage } from '../lib/mutation-error' +import { + SMART_OPS_ACTIVE_ALERTS_QUERY_KEY, + SMART_OPS_BILLING_RECONCILIATION_QUERY_KEY, +} from '../lib/query-keys' +import type { BillingSettlementReconciliationItem } from '../types' + +interface SettlementReviewDialogProps { + item: BillingSettlementReconciliationItem + open: boolean + onOpenChange: (open: boolean) => void +} + +export function SettlementReviewDialog( + props: SettlementReviewDialogProps +): ReactElement { + const { t, i18n } = useTranslation() + const [decision, setDecision] = useState<'block' | 'allow'>( + props.item.record_blocks_user ? 'block' : 'allow' + ) + const [note, setNote] = useState(props.item.reconciliation_review_note || '') + const queryClient = useQueryClient() + + const trimmedNote = note.trim() + const noteLength = Array.from(trimmedNote).length + const noteInvalid = noteLength < 3 || noteLength > 1000 + + const reviewMutation = useMutation({ + mutationKey: ['smart-ops', 'billing-settlement-review', props.item.id], + mutationFn: async () => { + const response = await reviewBillingSettlement(props.item.id, { + block_user: decision === 'block', + note: trimmedNote, + }) + if (!response.success) { + throw new Error(response.message || t('Failed to save review.')) + } + }, + onSuccess: async () => { + toast.success(t('Billing reconciliation alert closed.')) + props.onOpenChange(false) + await Promise.all([ + queryClient.invalidateQueries({ + queryKey: SMART_OPS_ACTIVE_ALERTS_QUERY_KEY, + }), + queryClient.invalidateQueries({ + queryKey: SMART_OPS_BILLING_RECONCILIATION_QUERY_KEY, + }), + ]) + }, + onError: (error) => { + handleServerError(error, { + fallback: mutationErrorMessage(error, t('Failed to save review.')), + }) + }, + }) + + const handleSubmit = () => { + if (noteLength < 3 || noteLength > 1000) return + reviewMutation.mutate() + } + + return ( + { + if (!reviewMutation.isPending) props.onOpenChange(open) + }} + > + + + + {props.item.reconciliation_reviewed_at + ? t('Edit reconciliation review') + : t('Review billing reconciliation')} + + + {t( + 'Closing this alert records an administrator review only. It does not mark the settlement as applied or change any balance.' + )} + + + + + + {t('User access while unresolved')} + { + const nextDecision = value[0] + if (nextDecision === 'block' || nextDecision === 'allow') { + setDecision(nextDecision) + } + }} + variant='outline' + spacing={2} + className='grid w-full grid-cols-1 sm:grid-cols-2' + > + + + + + + + {t( + 'This choice affects only new paid-request admission while the financial settlement remains pending or manual.' + )} + + + + + + {t('Review note')} + +