From 1bc3f4a07bc466d2724123f0e5a6484d126bf0a5 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Wed, 14 Oct 2026 04:01:37 +0800 Subject: [PATCH 01/16] test: verify pre3 SQLite startup and settlement recovery --- docs/installation/sqlite-upgrade.md | 25 ++ model/sqlite_upgrade_test.go | 504 +++++++++++++++++++++ model/testdata/sqlite/pre3_schema.sql | 625 ++++++++++++++++++++++++++ 3 files changed, 1154 insertions(+) create mode 100644 docs/installation/sqlite-upgrade.md create mode 100644 model/sqlite_upgrade_test.go create mode 100644 model/testdata/sqlite/pre3_schema.sql diff --git a/docs/installation/sqlite-upgrade.md b/docs/installation/sqlite-upgrade.md new file mode 100644 index 00000000..41e8146e --- /dev/null +++ b/docs/installation/sqlite-upgrade.md @@ -0,0 +1,25 @@ +# SQLite 升级验收 / SQLite upgrade acceptance + +升级前停止接收新请求，在原数据库完成待处理资金结算及日志副作用，停止服务后一起备份业务库、日志库和配置。SQLite 使用 WAL 时，保留尚未合并的 WAL 文件；运行中的单个数据库文件复制不能作为完整备份。 + +Before upgrading, stop new traffic, finish pending funding settlements and log effects against their original databases, stop the service, and back up the business database, log database, and configuration together. Preserve uncheckpointed WAL files. Copying only a database file while it is in use does not establish a complete backup. + +先对备份副本运行升级版本，核对钱包、Token、订阅用量与周期、历史日志、`billing_settlements` 和 `billing_log_receipts`，再重复启动并确认恢复没有再次扣费、退款或生成重复日志。待人工处理、未知结果或跨周期结算保留其核对状态，不能视为升级后自动成功。 + +Run the upgraded version against an isolated backup copy first. Check wallet and token balances, subscription usage and period, historical logs, `billing_settlements`, and `billing_log_receipts`. Restart again and confirm recovery does not debit, refund, or project a log twice. Manual, unknown-outcome, and cross-period records retain their reconciliation boundaries; upgrading does not imply their successful settlement. + +## 日志文件 / Log files + +默认同库和旧 `LOG_SQL_DSN=local` 可以继续使用。新增 `LOG_SQLITE_PATH` 时按[独立日志数据库指引](sqlite-log-database.md)准备目标文件；这个参数不复制或合并旧日志。完成原库的 pending effects，并保持日志和 receipt 一起保留，是切换前的必要准备。新空日志文件的启动验证只适用于已完成这些准备的场景。 + +Existing shared-database and `LOG_SQL_DSN=local` settings remain supported. Follow the [separate log database guide](sqlite-log-database.md) when configuring `LOG_SQLITE_PATH`. The setting does not copy or merge old logs. Finish effects against the original file and retain each log with its receipt before switching. Fresh-file startup acceptance assumes these preparations are complete. + +## 自动化覆盖 / Automated coverage + +`model/sqlite_upgrade_test.go` 使用 MAX-API `v2.0.0-smartops.pre3` 固定提交 `48260ce6517a9ab68851a7ac499477981d78da63` 的真实 `InitDB/InitLogDB` 生成 schema fixture（39 张表、160 个索引），并填入合成数据。当前版本在独立测试进程中执行主节点迁移及恢复，覆盖共享、旧 local、已有独立目标与完成结算后的新空目标，多次启动、64 位余额、明确零费用、原资金来源与周期、旧 receipt 去重及目标文件的日志标记回填。LF 和 Windows CRLF 格式均验证。fixture 只有 schema，不包含用户数据、密钥或运行时二进制。 + +The fixture comes from real `InitDB/InitLogDB` execution at the pinned MAX-API pre3 source revision: 39 tables and 160 indexes, with synthetic data added by the tests. Fresh test processes run current master migrations and recovery for shared, legacy-local, existing separate, and quiesced fresh log targets. Coverage includes repeated startup, 64-bit balances, explicit zero usage, original funding and periods, old receipt deduplication, and target-specific retry-marker backfill. Both LF and Windows CRLF inputs are checked. The committed fixture contains schema only, with no user data, credentials, or runtime binary. + +这个基线不等于验证了所有历史部署。更老版本、真实旧镜像与用户库、MySQL/PostgreSQL、混合引擎、Redis 多节点、在线切库、实盘掉电、保留清理策略及实际渠道仍需对应环境验收。升级后写入的数据不保证可直接降级；需要回退时使用成套升级前备份，并核对外部已接受请求和未完成结算。 + +This baseline does not establish compatibility with every historical deployment. Older releases, actual old images and user databases, MySQL/PostgreSQL, mixed engines, Redis clusters, online switching, power interruption, retention cleanup, and real channels require their own acceptance. New-version writes are not guaranteed to support direct downgrade. Restore the complete pre-upgrade backup for rollback and reconcile externally accepted requests and incomplete settlements. diff --git a/model/sqlite_upgrade_test.go b/model/sqlite_upgrade_test.go new file mode 100644 index 00000000..e58fba8e --- /dev/null +++ b/model/sqlite_upgrade_test.go @@ -0,0 +1,504 @@ +package model + +import ( + "context" + "crypto/sha256" + _ "embed" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/MAX-API-Next/MAX-API/common" + "github.com/glebarez/sqlite" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +// Generated by running the actual MAX-API pre3 InitDB/InitLogDB at +// 48260ce6517a9ab68851a7ac499477981d78da63, not by migrating current models. +// The fixture contains schema only. All data below is synthetic. +// +//go:embed testdata/sqlite/pre3_schema.sql +var sqlitePre3Schema string + +const sqlitePre3SchemaSHA = "f4af2569217ecb14863d551aae40b36d655e821a57503c9cdee5bef34e2a8a26" +const sqliteUpgradeConfigEnv = "MAXAPI_SQLITE_UPGRADE_TEST_CONFIG" + +type sqliteUpgradeConfig struct { + Directory string + Mode string + Recover bool +} + +type sqliteUpgradeCase struct { + ID int + Source, State string + Quota int64 +} + +func (item sqliteUpgradeCase) requestID() string { return fmt.Sprintf("synthetic-pre3-%d", item.ID) } +func (item sqliteUpgradeCase) tokenID() int { return item.ID + 1000 } +func (item sqliteUpgradeCase) subscriptionID() int { return item.ID + 2000 } +func (item sqliteUpgradeCase) operationKey() string { + return BillingRequestFinalizeOperationKey(item.requestID()) +} + +func sqliteUpgradeCases(mode string) []sqliteUpgradeCase { + var cases []sqliteUpgradeCase + for _, source := range []string{BillingSettlementSourceWallet, BillingSettlementSourceSubscription} { + for _, quota := range []int64{0, 6, 100, 130} { + states := []string{"pending", "effect_pending", "effect_logged", "complete"} + if mode == "separate_fresh" { + // A fresh target requires quiesced effects on the original file. + // This does not claim an online transfer of pending receipts. + states = []string{"complete"} + } + for _, state := range states { + cases = append(cases, sqliteUpgradeCase{ID: 8800 + len(cases), Source: source, State: state, Quota: quota}) + } + } + if mode != "separate_fresh" { + for _, state := range []string{"manual", "outcome_unknown"} { + cases = append(cases, sqliteUpgradeCase{ID: 8800 + len(cases), Source: source, State: state, Quota: 130}) + } + if source == BillingSettlementSourceSubscription { + cases = append(cases, sqliteUpgradeCase{ID: 8800 + len(cases), Source: source, State: "period_changed", Quota: 6}) + } + } + } + return cases +} + +func openSQLiteUpgradeFixture(t *testing.T, path string, logOnly bool) *gorm.DB { + return openSQLiteUpgradeSchema(t, path, logOnly, sqlitePre3Schema) +} + +func openSQLiteUpgradeSchema(t *testing.T, path string, logOnly bool, schema string) *gorm.DB { + t.Helper() + // Git checkouts can convert text fixtures to CRLF on Windows. + schema = strings.ReplaceAll(schema, "\r\n", "\n") + require.Equal(t, sqlitePre3SchemaSHA, fmt.Sprintf("%x", sha256.Sum256([]byte(schema))), "historical fixture cannot silently become a current-model schema") + db, err := gorm.Open(sqlite.Open(path), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + require.NoError(t, err) + require.NoError(t, db.Transaction(func(tx *gorm.DB) error { + for _, statement := range strings.Split(schema, "\n-- MAX_API_SQLITE_SCHEMA_STATEMENT\n")[1:] { + if logOnly && !strings.Contains(statement, "CREATE TABLE `logs`") && + !strings.Contains(statement, "CREATE TABLE `billing_log_receipts`") && + !strings.Contains(statement, " ON `logs`") && !strings.Contains(statement, " ON `billing_log_receipts`") { + continue + } + if err := tx.Exec(statement).Error; err != nil { + return err + } + } + return nil + })) + return db +} + +func TestSQLitePre3FixtureFormat(t *testing.T) { + for _, format := range []string{"LF", "CRLF"} { + t.Run(format, func(t *testing.T) { + schema := strings.ReplaceAll(sqlitePre3Schema, "\r\n", "\n") + if format == "CRLF" { + schema = strings.ReplaceAll(schema, "\n", "\r\n") + } + db := openSQLiteUpgradeSchema(t, filepath.Join(t.TempDir(), "fixture.sqlite"), false, schema) + defer func() { require.NoError(t, closeDB(db)) }() + var tables, indexes int64 + require.NoError(t, db.Raw("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'").Scan(&tables).Error) + require.NoError(t, db.Raw("SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name NOT LIKE 'sqlite_%'").Scan(&indexes).Error) + require.EqualValues(t, 39, tables) + require.EqualValues(t, 160, indexes) + }) + } +} + +func seedSQLiteUpgradeFixture(t *testing.T, mode string) string { + t.Helper() + directory := filepath.Join(t.TempDir(), "upgrade 中文 space") + require.NoError(t, os.MkdirAll(directory, 0755)) + db := openSQLiteUpgradeFixture(t, filepath.Join(directory, "ledger.sqlite"), false) + t.Cleanup(func() { _ = closeDB(db) }) + logDB := db + if mode == "separate_existing" { + logDB = openSQLiteUpgradeFixture(t, filepath.Join(directory, "logs.sqlite"), true) + t.Cleanup(func() { _ = closeDB(logDB) }) + require.False(t, logDB.Migrator().HasTable(&User{})) + } + // Seed values directly into the captured schema without running current + // save hooks or creating current tables before the upgrade under test. + seed := db.Session(&gorm.Session{SkipHooks: true}) + require.NoError(t, seed.Create(&User{Id: 8700, Username: "synthetic-pre3-history", AffCode: "uphist", Email: "Legacy@Example.invalid", Quota: 3000000000, + UsedQuota: 4000000000, RequestCount: 7, Status: common.UserStatusEnabled}).Error) + require.NoError(t, seed.Create(&Token{Id: 8701, UserId: 8700, Key: "synthetic-pre3-history", RemainQuota: 5000000000, + UsedQuota: 4000000000, Status: common.TokenStatusEnabled}).Error) + require.NoError(t, seed.Create(&Channel{Id: 8702, UsedQuota: 4000000000}).Error) + history := Log{Id: 8703, UserId: 8700, Type: LogTypeConsume, Quota: 23, PromptTokens: 10, CompletionTokens: 13, + Content: "synthetic 历史记录", RequestId: "synthetic-pre3-history", Other: `{"retry_log":true}`} + require.NoError(t, seed.Create(&history).Error) + require.NoError(t, seed.Create(&BillingLogReceipt{ID: 8704, OperationKey: "synthetic-pre3-history", ClaimToken: "synthetic-history", CreatedAt: 1700000000}).Error) + if mode == "separate_existing" { + target := logDB.Session(&gorm.Session{SkipHooks: true}) + require.NoError(t, target.Create(&Log{Id: 8703, UserId: 8700, Type: LogTypeConsume, Quota: 23, + Content: "synthetic 独立历史日志", RequestId: "synthetic-pre3-target-history", Other: `{"retry_log":true}`}).Error) + require.NoError(t, target.Create(&BillingLogReceipt{ID: 8704, OperationKey: "synthetic-pre3-target-history", + ClaimToken: "synthetic-target-history", CreatedAt: 1700000000}).Error) + } + + for _, item := range sqliteUpgradeCases(mode) { + finalized := item.State == "effect_pending" || item.State == "effect_logged" || item.State == "complete" + amount := int64(100) + if finalized { + amount = item.Quota + } + wallet := int64(1000) + if item.Source == BillingSettlementSourceWallet { + wallet -= amount + } + used, requests := int64(0), 0 + if item.State == "complete" { + used, requests = item.Quota, 1 + } + require.NoError(t, seed.Create(&User{Id: item.ID, Username: item.requestID(), AffCode: fmt.Sprintf("up%d", item.ID), Quota: wallet, UsedQuota: used, + RequestCount: requests, Status: common.UserStatusEnabled}).Error) + require.NoError(t, seed.Create(&Token{Id: item.tokenID(), UserId: item.ID, Key: item.requestID(), + RemainQuota: 1000 - amount, UsedQuota: amount, Status: common.TokenStatusEnabled}).Error) + require.NoError(t, seed.Create(&Channel{Id: item.ID, UsedQuota: used}).Error) + if item.Source == BillingSettlementSourceSubscription { + allow := false + require.NoError(t, seed.Create(&SubscriptionPlan{Id: item.ID, Title: item.requestID(), Enabled: true, + TotalAmount: 1000, AllowWalletOverflow: &allow, QuotaResetPeriod: SubscriptionResetNever}).Error) + period := int64(1700000000) + if item.State == "period_changed" { + period++ + } + require.NoError(t, seed.Create(&UserSubscription{Id: item.subscriptionID(), UserId: item.ID, PlanId: item.ID, + AmountTotal: 1000, AmountUsed: amount, Status: "active", StartTime: 1700000000, + EndTime: 9999999999, LastResetTime: period, AllowWalletOverflow: false}).Error) + require.NoError(t, seed.Create(&SubscriptionPreConsumeRecord{RequestId: item.requestID(), UserId: item.ID, + TokenId: item.tokenID(), UserSubscriptionId: item.subscriptionID(), SubscriptionLastResetTime: 1700000000, + ModelName: "synthetic-upgrade-model", PreConsumed: 100, Status: "consumed", CreatedAt: 1700000001, UpdatedAt: 1700000001}).Error) + } + require.NoError(t, seed.Create(&BillingPreConsumeSelection{RequestID: item.requestID(), Source: item.Source, + UserID: item.ID, TokenID: item.tokenID(), ModelName: "synthetic-upgrade-model", RequestedQuota: 100, + EffectiveQuota: 100, CreatedAt: 1700000001}).Error) + prompt, completion := int(item.Quota), 0 + if item.Quota > 0 { + prompt, completion = int(item.Quota)-2, 2 + } + effect := BillingSettlementEffect{LogType: LogTypeConsume, Content: "synthetic pre3 recovery", ChannelID: item.ID, + ModelName: "synthetic-upgrade-model", TokenID: item.tokenID(), TokenName: "synthetic-upgrade-token", Group: "default", + UpdateUsage: true, Quota: item.Quota, QuotaIsActual: true, PromptTokens: prompt, CompletionTokens: completion, RequestID: item.requestID()} + payload, err := common.Marshal(effect) + require.NoError(t, err) + record := BillingSettlement{OperationKey: item.operationKey(), Source: item.Source, UserID: item.ID, TokenID: item.tokenID(), + FundingDelta: item.Quota - 100, TokenDelta: item.Quota - 100, Status: BillingSettlementStatusPending, + Revision: 1, EffectPayload: string(payload), EffectStatus: BillingSettlementEffectPending, CreatedAt: 1700000002, UpdatedAt: 1700000002} + if item.Source == BillingSettlementSourceSubscription { + record.SubscriptionID, record.SubscriptionPreConsumeRequestID = item.subscriptionID(), item.requestID() + } + if finalized { + record.Status = BillingSettlementStatusApplied + record.AppliedFundingDelta, record.AppliedTokenDelta = record.FundingDelta, record.TokenDelta + } + if item.State == "manual" || item.State == "outcome_unknown" { + record.Status = item.State + } + if item.State == "complete" { + record.EffectStatus = BillingSettlementEffectApplied + } + require.NoError(t, seed.Create(&record).Error) + if item.State == "effect_logged" || item.State == "complete" { + logs := logDB.Session(&gorm.Session{SkipHooks: true}) + require.NoError(t, logs.Create(&Log{UserId: item.ID, Type: LogTypeConsume, Quota: int(item.Quota), PromptTokens: prompt, + CompletionTokens: completion, ModelName: effect.ModelName, TokenId: item.tokenID(), RequestId: item.requestID()}).Error) + require.NoError(t, logs.Create(&BillingLogReceipt{OperationKey: item.operationKey(), ClaimToken: "synthetic-old-receipt", CreatedAt: 1700000003}).Error) + } + } + require.NoError(t, closeDB(logDB)) + if logDB != db { + require.NoError(t, closeDB(db)) + } + require.NoError(t, os.WriteFile(filepath.Join(directory, ".synthetic-upgrade-fixture"), []byte(sqlitePre3SchemaSHA), 0600)) + return directory +} + +// A cold process runs the real master startup with isolated globals and +// connections. The parent never substitutes a fresh AutoMigrate schema. +func TestSQLiteUpgradeProcessHelper(t *testing.T) { + encoded := os.Getenv(sqliteUpgradeConfigEnv) + if encoded == "" { + t.Skip("cold-start helper invoked by TestSQLitePre3Upgrade") + } + var config sqliteUpgradeConfig + require.NoError(t, common.UnmarshalJsonStr(encoded, &config)) + marker, err := os.ReadFile(filepath.Join(config.Directory, ".synthetic-upgrade-fixture")) + require.NoError(t, err) + require.Equal(t, sqlitePre3SchemaSHA, string(marker)) + require.Contains(t, []string{"shared", "legacy_local", "separate_existing", "separate_fresh"}, config.Mode) + t.Setenv("SQL_DSN", "local") + t.Setenv("LOG_SQL_DSN", "") + t.Setenv("LOG_SQLITE_PATH", "") + t.Setenv("SQL_MAX_IDLE_CONNS", "1") + t.Setenv("SQL_MAX_OPEN_CONNS", "1") + t.Setenv("QUOTA_DATA_AGGREGATE_MIGRATION_ENABLED", "false") + if config.Mode == "legacy_local" { + t.Setenv("LOG_SQL_DSN", "local") + } + if strings.HasPrefix(config.Mode, "separate_") { + t.Setenv("LOG_SQLITE_PATH", filepath.Join(config.Directory, "logs.sqlite")) + } + common.SQLitePath = filepath.Join(config.Directory, "ledger.sqlite") + common.IsMasterNode, common.UsingSQLite = true, true + common.UsingMySQL, common.UsingPostgreSQL = false, false + common.RedisEnabled, common.BatchUpdateEnabled, common.DataExportEnabled = false, false, false + common.LogConsumeEnabled, common.DebugEnabled = true, false + logRetryMarkerBackfillAsyncRunner = func(fn func()) { fn() } + require.NoError(t, InitDB()) + t.Cleanup(func() { require.NoError(t, CloseDB()) }) + require.NoError(t, InitLogDB()) + DB.Logger, LOG_DB.Logger = logger.Default.LogMode(logger.Silent), logger.Default.LogMode(logger.Silent) + var history User + require.NoError(t, DB.First(&history, 8700).Error) + require.Equal(t, "synthetic-pre3-history", history.Username) + if config.Recover { + ProcessPendingBillingSettlementsOnce() + ProcessPendingBillingSettlementsOnce() + for _, item := range sqliteUpgradeCases(config.Mode) { + var record BillingSettlement + require.NoError(t, DB.Where("operation_key = ?", item.operationKey()).First(&record).Error) + if item.State == "manual" || item.State == "outcome_unknown" || item.State == "period_changed" { + continue + } + var effect BillingSettlementEffect + require.NoError(t, common.UnmarshalJsonStr(record.EffectPayload, &effect)) + for range 2 { + applied, duplicate, err := ApplyBillingSettlementOnce(BillingSettlementInput{OperationKey: record.OperationKey, + Source: record.Source, UserID: record.UserID, SubscriptionID: record.SubscriptionID, TokenID: record.TokenID, + FundingDelta: record.FundingDelta, TokenDelta: record.TokenDelta, + SubscriptionPreConsumeRequestID: record.SubscriptionPreConsumeRequestID, Effect: &effect}) + require.NoError(t, err) + require.True(t, duplicate) + require.Equal(t, item.Quota-100, applied) + require.NoError(t, ProcessBillingSettlementEffect(record.OperationKey)) + } + } + } +} + +func runSQLiteUpgradeProcess(t *testing.T, config sqliteUpgradeConfig) { + t.Helper() + encoded, err := common.Marshal(config) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestSQLiteUpgradeProcessHelper$", "-test.count=1", "-test.timeout=40s", "-test.v") + for _, entry := range os.Environ() { + if !strings.HasPrefix(entry, sqliteUpgradeConfigEnv+"=") { + command.Env = append(command.Env, entry) + } + } + command.Env = append(command.Env, sqliteUpgradeConfigEnv+"="+string(encoded)) + output, err := command.CombinedOutput() + require.NoError(t, err, "cold startup/recovery: %s", output) +} + +func sqliteUpgradeFinancialSnapshot(t *testing.T, directory string) string { + t.Helper() + db, err := gorm.Open(sqlite.Open(filepath.Join(directory, "ledger.sqlite")), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + require.NoError(t, err) + defer func() { require.NoError(t, closeDB(db)) }() + var users []struct { + Id int + Quota, UsedQuota int64 + RequestCount int + } + var tokens []Token + var subscriptions []UserSubscription + var selections []BillingPreConsumeSelection + var reservations []SubscriptionPreConsumeRecord + var settlements []BillingSettlement + var channels []struct { + Id int + UsedQuota int64 + } + require.NoError(t, db.Model(&User{}).Order("id").Find(&users).Error) + require.NoError(t, db.Order("id").Find(&tokens).Error) + require.NoError(t, db.Order("id").Find(&subscriptions).Error) + require.NoError(t, db.Order("id").Find(&selections).Error) + require.NoError(t, db.Order("id").Find(&reservations).Error) + require.NoError(t, db.Order("id").Find(&settlements).Error) + require.NoError(t, db.Model(&Channel{}).Order("id").Find(&channels).Error) + data, err := common.Marshal([]any{users, tokens, subscriptions, selections, reservations, settlements, channels}) + require.NoError(t, err) + return string(data) +} + +func assertSQLiteUpgradedRecovery(t *testing.T, directory, mode string) { + t.Helper() + db, err := gorm.Open(sqlite.Open(filepath.Join(directory, "ledger.sqlite")), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + require.NoError(t, err) + defer func() { require.NoError(t, closeDB(db)) }() + logDB := db + if strings.HasPrefix(mode, "separate_") { + logDB, err = gorm.Open(sqlite.Open(filepath.Join(directory, "logs.sqlite")), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + require.NoError(t, err) + defer func() { require.NoError(t, closeDB(logDB)) }() + require.False(t, logDB.Migrator().HasTable(&User{})) + require.False(t, logDB.Migrator().HasTable(&BillingSettlement{})) + } + for _, item := range sqliteUpgradeCases(mode) { + t.Run(fmt.Sprintf("%s/%s/quota%d", item.Source, item.State, item.Quota), func(t *testing.T) { + blocked := item.State == "manual" || item.State == "outcome_unknown" || item.State == "period_changed" + amount := item.Quota + if blocked { + amount = 100 + } + var user User + var token Token + var channel Channel + var record BillingSettlement + require.NoError(t, db.First(&user, item.ID).Error) + require.NoError(t, db.First(&token, item.tokenID()).Error) + require.NoError(t, db.First(&channel, item.ID).Error) + require.NoError(t, db.Where("operation_key = ?", item.operationKey()).First(&record).Error) + wallet := int64(1000) + if item.Source == BillingSettlementSourceWallet { + wallet -= amount + } + require.Equal(t, wallet, user.Quota) + require.Equal(t, 1000-amount, token.RemainQuota) + require.Equal(t, amount, token.UsedQuota) + require.Equal(t, item.Source, record.Source) + require.Equal(t, item.ID, record.UserID) + require.Equal(t, item.tokenID(), record.TokenID) + if item.Source == BillingSettlementSourceSubscription { + var subscription UserSubscription + require.NoError(t, db.First(&subscription, item.subscriptionID()).Error) + require.Equal(t, amount, subscription.AmountUsed) + require.False(t, subscription.AllowWalletOverflow) + var reservation SubscriptionPreConsumeRecord + require.NoError(t, db.Where("request_id = ?", item.requestID()).First(&reservation).Error) + require.EqualValues(t, 1700000000, reservation.SubscriptionLastResetTime) + require.EqualValues(t, 100, reservation.PreConsumed) + require.Equal(t, item.subscriptionID(), record.SubscriptionID) + } + if blocked { + want := item.State + if want == "period_changed" { + want = BillingSettlementStatusManual + } + require.Equal(t, want, record.Status) + require.Zero(t, record.AppliedFundingDelta) + require.Zero(t, record.AppliedTokenDelta) + require.Zero(t, user.UsedQuota) + require.Zero(t, user.RequestCount) + require.Zero(t, channel.UsedQuota) + } else { + require.Equal(t, BillingSettlementStatusApplied, record.Status) + require.Equal(t, BillingSettlementEffectApplied, record.EffectStatus) + require.Equal(t, item.Quota-100, record.AppliedFundingDelta) + require.Equal(t, item.Quota-100, record.AppliedTokenDelta) + require.Equal(t, item.Quota, user.UsedQuota) + require.Equal(t, 1, user.RequestCount) + require.Equal(t, item.Quota, channel.UsedQuota) + } + var logs, receipts int64 + require.NoError(t, logDB.Model(&Log{}).Where("request_id = ?", item.requestID()).Count(&logs).Error) + require.NoError(t, logDB.Model(&BillingLogReceipt{}).Where("operation_key = ?", item.operationKey()).Count(&receipts).Error) + want := int64(1) + if blocked || mode == "separate_fresh" { + want = 0 + } + require.Equal(t, want, logs) + require.Equal(t, want, receipts) + if want == 1 { + var log Log + require.NoError(t, logDB.Where("request_id = ?", item.requestID()).First(&log).Error) + require.EqualValues(t, item.Quota, log.Quota) + require.EqualValues(t, item.Quota, log.PromptTokens+log.CompletionTokens) + if item.State == "effect_logged" || item.State == "complete" { + var receipt BillingLogReceipt + require.NoError(t, logDB.Where("operation_key = ?", item.operationKey()).First(&receipt).Error) + require.Equal(t, "synthetic-old-receipt", receipt.ClaimToken) + require.EqualValues(t, 1700000003, receipt.CreatedAt) + } + } + }) + } + var history User + require.NoError(t, db.First(&history, 8700).Error) + require.EqualValues(t, 3000000000, history.Quota) + require.EqualValues(t, 4000000000, history.UsedQuota) + require.Equal(t, 7, history.RequestCount) + require.Equal(t, "Legacy@Example.invalid", history.Email) + require.Equal(t, "legacy@example.invalid", history.NormalizedEmail) + var historyLog Log + require.NoError(t, db.First(&historyLog, 8703).Error) + require.Equal(t, "synthetic 历史记录", historyLog.Content) + require.Equal(t, 23, historyLog.Quota) + require.Equal(t, !strings.HasPrefix(mode, "separate_"), historyLog.IsRetry) + var receipt BillingLogReceipt + require.NoError(t, db.First(&receipt, 8704).Error) + require.Equal(t, "synthetic-pre3-history", receipt.OperationKey) + require.Equal(t, "synthetic-history", receipt.ClaimToken) + var options []Option + require.NoError(t, db.Find(&options).Error) + var completed []Option + for _, option := range options { + if strings.HasPrefix(option.Key, logRetryMarkerBackfillOptionKey) { + completed = append(completed, option) + } + } + require.Len(t, completed, 1, "repeated startup retains one completion identity for the active file") + require.Equal(t, "true", completed[0].Value) + if mode == "shared" { + require.Equal(t, logRetryMarkerBackfillOptionKey, completed[0].Key) + } else { + require.NotEqual(t, logRetryMarkerBackfillOptionKey, completed[0].Key) + } + if mode == "separate_existing" { + var targetHistory Log + require.NoError(t, logDB.First(&targetHistory, 8703).Error) + require.Equal(t, "synthetic 独立历史日志", targetHistory.Content) + require.True(t, targetHistory.IsRetry) + require.True(t, targetHistory.IsErrorRetry) + require.NoError(t, logDB.First(&receipt, 8704).Error) + require.Equal(t, "synthetic-pre3-target-history", receipt.OperationKey) + require.Equal(t, "synthetic-target-history", receipt.ClaimToken) + } + if mode == "separate_fresh" { + var logs, receipts int64 + require.NoError(t, logDB.Model(&Log{}).Count(&logs).Error) + require.NoError(t, logDB.Model(&BillingLogReceipt{}).Count(&receipts).Error) + require.Zero(t, logs, "fresh target does not implicitly copy old history") + require.Zero(t, receipts) + } +} + +func TestSQLitePre3Upgrade(t *testing.T) { + for _, mode := range []string{"shared", "legacy_local", "separate_existing", "separate_fresh"} { + t.Run(mode, func(t *testing.T) { + directory := seedSQLiteUpgradeFixture(t, mode) + before := sqliteUpgradeFinancialSnapshot(t, directory) + config := sqliteUpgradeConfig{Directory: directory, Mode: mode} + for range 2 { + runSQLiteUpgradeProcess(t, config) + require.Equal(t, before, sqliteUpgradeFinancialSnapshot(t, directory), "startup migration must not debit/refund or change a durable intent") + } + config.Recover = true + runSQLiteUpgradeProcess(t, config) + assertSQLiteUpgradedRecovery(t, directory, mode) + settled := sqliteUpgradeFinancialSnapshot(t, directory) + runSQLiteUpgradeProcess(t, config) + assertSQLiteUpgradedRecovery(t, directory, mode) + require.Equal(t, settled, sqliteUpgradeFinancialSnapshot(t, directory), "fresh-process recovery returns recorded outcomes without another mutation") + }) + } +} diff --git a/model/testdata/sqlite/pre3_schema.sql b/model/testdata/sqlite/pre3_schema.sql new file mode 100644 index 00000000..96211b48 --- /dev/null +++ b/model/testdata/sqlite/pre3_schema.sql @@ -0,0 +1,625 @@ +-- MAX-API SQLite schema generated by real pre3 InitDB/InitLogDB. +-- Source: v2.0.0-smartops.pre3 48260ce6517a9ab68851a7ac499477981d78da63 +-- Schema only; no user data or credentials. + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `abilities` (`group` varchar(64),`model` varchar(255),`channel_id` integer,`enabled` numeric,`priority` integer DEFAULT 0,`weight` integer DEFAULT 0,`tag` text,PRIMARY KEY (`group`,`model`,`channel_id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `auth_flows` (`id` integer,`token_hash` char(64) NOT NULL,`purpose` varchar(32) NOT NULL,`provider` varchar(64),`intent` varchar(16),`user_id` integer,`session_id` varchar(64),`payload` text,`created_at` datetime,`expires_at` datetime NOT NULL,`consumed_at` datetime,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `authz_user_overrides` (`id` integer,`user_id` integer NOT NULL,`resource` varchar(64) NOT NULL,`action` varchar(64) NOT NULL,`allowed` numeric NOT NULL,`created_at` integer NOT NULL,`updated_at` integer NOT NULL,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `billing_log_receipts` (`id` integer,`operation_key` varchar(191) NOT NULL,`claim_token` varchar(36) NOT NULL DEFAULT "",`created_at` integer NOT NULL,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `billing_pre_consume_selections` (`id` integer,`request_id` varchar(64) NOT NULL,`source` varchar(32) NOT NULL,`user_id` integer NOT NULL,`token_id` integer NOT NULL DEFAULT 0,`model_name` varchar(191) NOT NULL DEFAULT "",`requested_quota` integer NOT NULL DEFAULT 0,`effective_quota` integer NOT NULL DEFAULT 0,`created_at` integer NOT NULL,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `billing_settlements` (`id` integer,`operation_key` varchar(191) NOT NULL,`source` varchar(32) NOT NULL,`user_id` integer NOT NULL DEFAULT 0,`subscription_id` integer NOT NULL DEFAULT 0,`token_id` integer NOT NULL DEFAULT 0,`funding_delta` integer NOT NULL DEFAULT 0,`applied_funding_delta` integer NOT NULL DEFAULT 0,`token_delta` integer NOT NULL DEFAULT 0,`applied_token_delta` integer NOT NULL DEFAULT 0,`task_id` integer NOT NULL DEFAULT 0,`task_quota` integer NOT NULL DEFAULT 0,`task_quota_target` integer NOT NULL DEFAULT 0,`status` varchar(16) NOT NULL DEFAULT "applied",`attempts` integer NOT NULL DEFAULT 0,`last_error` text,`next_attempt` integer NOT NULL DEFAULT 0,`revision` integer NOT NULL DEFAULT 1,`subscription_pre_consume_request_id` varchar(64),`finalize_subscription_pre_consume` numeric NOT NULL DEFAULT false,`allow_missing_token` numeric NOT NULL DEFAULT false,`manual_on_failure` numeric NOT NULL DEFAULT false,`pre_consume_request_id` varchar(64),`pre_consume_model_name` varchar(191) NOT NULL DEFAULT "",`pre_consume_requested_quota` integer NOT NULL DEFAULT 0,`pre_consume_effective_quota` integer NOT NULL DEFAULT 0,`effect_payload` text,`effect_status` varchar(16),`reconciliation_reviewed_at` integer NOT NULL DEFAULT 0,`reconciliation_reviewed_by` integer NOT NULL DEFAULT 0,`reconciliation_review_note` text,`user_blocking_override` numeric,`created_at` integer NOT NULL,`updated_at` integer NOT NULL,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `cache_invalidation_tasks` (`id` integer,`kind` varchar(16) NOT NULL,`entity_key` varchar(96) NOT NULL,`version_key` varchar(96) NOT NULL,`delete_entry` numeric NOT NULL DEFAULT false,`attempts` integer NOT NULL DEFAULT 0,`last_error` text,`next_attempt` integer NOT NULL DEFAULT 0,`created_at` integer NOT NULL,`updated_at` integer NOT NULL,`revision` integer NOT NULL DEFAULT 1,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `channels` (`id` integer,`type` integer DEFAULT 0,`key` text NOT NULL,`open_ai_organization` text,`test_model` text,`status` integer DEFAULT 1,`name` text,`weight` integer DEFAULT 0,`created_time` integer,`test_time` integer,`response_time` integer,`base_url` text DEFAULT "",`other` text,`balance` real,`balance_updated_time` integer,`models` text,`group` varchar(64) DEFAULT "default",`used_quota` integer DEFAULT 0,`model_mapping` text,`status_code_mapping` varchar(1024) DEFAULT "",`priority` integer DEFAULT 0,`auto_ban` integer DEFAULT 1,`other_info` text,`tag` text,`setting` text,`param_override` text,`header_override` text,`remark` varchar(255),`channel_info` json,`settings` text,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `checkins` (`id` integer,`user_id` integer NOT NULL,`checkin_date` varchar(10) NOT NULL,`quota_awarded` integer NOT NULL,`created_at` integer,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `custom_oauth_providers` (`id` integer,`name` varchar(64) NOT NULL,`slug` varchar(64) NOT NULL,`icon` varchar(128) DEFAULT "",`enabled` numeric DEFAULT false,`client_id` varchar(256),`client_secret` varchar(512),`authorization_endpoint` varchar(512),`token_endpoint` varchar(512),`user_info_endpoint` varchar(512),`scopes` varchar(256) DEFAULT "openid profile email",`user_id_field` varchar(128) DEFAULT "sub",`username_field` varchar(128) DEFAULT "preferred_username",`display_name_field` varchar(128) DEFAULT "name",`email_field` varchar(128) DEFAULT "email",`well_known` varchar(512),`auth_style` integer DEFAULT 0,`access_policy` text,`access_denied_message` varchar(512),`created_at` datetime,`updated_at` datetime,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `logs` (`id` integer,`user_id` integer,`created_at` integer,`type` integer,`content` text,`username` text DEFAULT "",`token_name` text DEFAULT "",`model_name` text DEFAULT "",`quota` integer DEFAULT 0,`prompt_tokens` integer DEFAULT 0,`completion_tokens` integer DEFAULT 0,`use_time` integer DEFAULT 0,`is_stream` numeric,`channel_id` integer,`channel_name` text,`token_id` integer DEFAULT 0,`group` text,`ip` text DEFAULT "",`request_id` varchar(64) DEFAULT "",`upstream_request_id` varchar(128) DEFAULT "",`other` text,`is_retry` numeric DEFAULT false,`is_error_retry` numeric DEFAULT false,`is_empty_retry` numeric DEFAULT false,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `midjourney_billing_claims` (`id` integer,`channel_id` integer NOT NULL,`mj_id` varchar(180) NOT NULL,`user_id` integer NOT NULL,`billing_task_id` integer NOT NULL DEFAULT 0,`billing_request_id` varchar(64) NOT NULL DEFAULT "",`created_at` integer NOT NULL,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `midjourneys` (`id` integer,`code` integer,`user_id` integer,`action` varchar(40),`mj_id` text,`prompt` text,`prompt_en` text,`description` text,`state` text,`submit_time` integer,`start_time` integer,`finish_time` integer,`image_url` text,`video_url` text,`video_urls` text,`status` varchar(20),`progress` varchar(30),`fail_reason` text,`channel_id` integer,`quota` integer,`buttons` text,`properties` text,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `models` (`id` integer,`model_name` text NOT NULL,`name_key` text NOT NULL DEFAULT "",`description` text,`icon` varchar(128),`tags` varchar(255),`vendor_id` integer,`endpoints` text,`status` integer DEFAULT 1,`sync_official` integer DEFAULT 1,`created_time` integer,`updated_time` integer,`deleted_at` datetime,`name_rule` integer DEFAULT 0,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `options` (`key` text,`value` text,PRIMARY KEY (`key`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `passkey_credentials` (`id` integer,`user_id` integer NOT NULL,`credential_id` varchar(512) NOT NULL,`public_key` text NOT NULL,`attestation_type` varchar(255),`aa_guid` varchar(512),`sign_count` integer DEFAULT 0,`clone_warning` numeric,`user_present` numeric,`user_verified` numeric,`backup_eligible` numeric,`backup_state` numeric,`transports` text,`attachment` varchar(32),`last_used_at` datetime,`created_at` datetime,`updated_at` datetime,`deleted_at` datetime,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `perf_metric_flush_receipts` (`id` integer,`receipt_key` text NOT NULL,`claim_token` text NOT NULL,`created_at` integer NOT NULL,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `perf_metrics` (`id` integer,`model_name` text,`group` text,`bucket_ts` integer,`request_count` integer DEFAULT 0,`success_count` integer DEFAULT 0,`total_latency_ms` integer DEFAULT 0,`ttft_sum_ms` integer DEFAULT 0,`ttft_count` integer DEFAULT 0,`output_tokens` integer DEFAULT 0,`generation_ms` integer DEFAULT 0,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `prefill_groups` (`id` integer,`name` text NOT NULL,`type` text NOT NULL,`items` json,`description` varchar(255),`created_time` integer,`updated_time` integer,`deleted_at` datetime,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `quota_data` (`id` integer,`aggregate_key` varchar(64),`user_id` integer,`username` text DEFAULT "",`model_name` text DEFAULT "",`created_at` integer,`use_group` text DEFAULT "",`token_id` integer DEFAULT 0,`channel_id` integer DEFAULT 0,`node_name` text DEFAULT "",`token_used` integer DEFAULT 0,`count` integer DEFAULT 0,`quota` integer DEFAULT 0,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `quota_data_operation_locks` (`name` varchar(64) PRIMARY KEY, `owner` varchar(64) NOT NULL, `expires_at` bigint NOT NULL, `created_at` bigint NOT NULL, `updated_at` bigint NOT NULL); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `quota_data_snapshot_retries` (`snapshot_id` varchar(64),`retry_until` integer,`updated_at` integer,PRIMARY KEY (`snapshot_id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `quota_data_snapshots` (`snapshot_id` varchar(64),`created_at` integer,PRIMARY KEY (`snapshot_id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `redemptions` (`id` integer,`user_id` integer,`key` char(32),`status` integer DEFAULT 1,`name` text,`quota` integer DEFAULT 100,`created_time` integer,`redeemed_time` integer,`used_user_id` integer,`deleted_at` datetime,`expired_time` integer,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `setups` (`id` integer,`version` varchar(50) NOT NULL,`initialized_at` bigint NOT NULL,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `subscription_orders` (`id` integer,`user_id` integer,`plan_id` integer,`money` real,`trade_no` varchar(255) UNIQUE,`payment_method` varchar(50),`payment_provider` varchar(50) DEFAULT "",`status` text,`create_time` integer,`complete_time` integer,`provider_payload` text,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `subscription_plans` ( +`id` integer, +`title` varchar(128) NOT NULL, +`subtitle` varchar(255) DEFAULT '', +`price_amount` decimal(10,6) NOT NULL, +`currency` varchar(8) NOT NULL DEFAULT 'USD', +`duration_unit` varchar(16) NOT NULL DEFAULT 'month', +`duration_value` integer NOT NULL DEFAULT 1, +`custom_seconds` bigint NOT NULL DEFAULT 0, +`enabled` numeric DEFAULT 1, +`sort_order` integer DEFAULT 0, +`allow_balance_pay` numeric DEFAULT 1, +`allow_wallet_overflow` numeric DEFAULT 1, +`stripe_price_id` varchar(128) DEFAULT '', +`creem_product_id` varchar(128) DEFAULT '', +`waffo_pancake_product_id` varchar(128) DEFAULT '', +`max_purchase_per_user` integer DEFAULT 0, +`upgrade_group` varchar(64) DEFAULT '', +`downgrade_group` varchar(64) DEFAULT '', +`total_amount` bigint NOT NULL DEFAULT 0, +`quota_reset_period` varchar(16) DEFAULT 'never', +`quota_reset_custom_seconds` bigint DEFAULT 0, +`created_at` bigint, +`updated_at` bigint, +PRIMARY KEY (`id`) +); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `subscription_pre_consume_records` (`id` integer,`request_id` varchar(64),`user_id` integer,`token_id` integer,`user_subscription_id` integer,`subscription_last_reset_time` integer NOT NULL DEFAULT 0,`model_name` varchar(191),`quota_type` integer,`pre_consumed` bigint NOT NULL DEFAULT 0,`status` varchar(32),`created_at` integer,`updated_at` integer,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `system_instances` (`node_name` varchar(128),`info` text,`started_at` integer,`last_seen_at` integer,`created_at` integer,`updated_at` integer,PRIMARY KEY (`node_name`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `system_tasks` (`id` integer,`task_id` varchar(64),`type` varchar(64),`status` varchar(32),`active_key` varchar(64),`payload` text,`state` text,`result` text,`error` text,`locked_by` varchar(128),`locked_until` integer,`created_at` integer,`updated_at` integer,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `tasks` (`id` integer,`created_at` integer,`updated_at` integer,`task_id` varchar(191),`platform` varchar(30),`user_id` integer,`group` varchar(50),`channel_id` integer,`quota` integer,`action` varchar(40),`status` varchar(20),`fail_reason` text,`submit_time` integer,`start_time` integer,`finish_time` integer,`progress` varchar(20),`properties` json,`private_data` json,`data` json,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `tokens` (`id` integer,`user_id` integer,`key` varchar(128),`status` integer DEFAULT 1,`name` text,`created_time` integer,`accessed_time` integer,`expired_time` integer DEFAULT -1,`remain_quota` bigint DEFAULT 0,`unlimited_quota` numeric,`model_limits_enabled` numeric,`model_limits` text,`allow_ips` text DEFAULT "",`used_quota` bigint DEFAULT 0,`group` text DEFAULT "",`cross_group_retry` numeric,`routing_policy` text,`deleted_at` datetime,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `top_ups` (`id` integer,`user_id` integer,`amount` integer,`money` real,`trade_no` varchar(255) UNIQUE,`payment_method` varchar(50),`payment_provider` varchar(50) DEFAULT "",`create_time` integer,`complete_time` integer,`status` text,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `two_fa_backup_codes` (`id` integer,`user_id` integer NOT NULL,`code_hash` varchar(255) NOT NULL,`is_used` numeric,`used_at` datetime,`created_at` datetime,`deleted_at` datetime,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `two_fas` (`id` integer,`user_id` integer NOT NULL UNIQUE,`secret` varchar(255) NOT NULL,`is_enabled` numeric,`failed_attempts` integer DEFAULT 0,`locked_until` datetime,`last_used_at` datetime,`created_at` datetime,`updated_at` datetime,`deleted_at` datetime,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `user_oauth_bindings` (`id` integer,`user_id` integer NOT NULL,`provider_id` integer NOT NULL,`provider_user_id` varchar(256) NOT NULL,`created_at` datetime,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `user_subscriptions` (`id` integer,`user_id` integer,`plan_id` integer,`amount_total` bigint NOT NULL DEFAULT 0,`amount_used` bigint NOT NULL DEFAULT 0,`start_time` integer,`end_time` integer,`status` varchar(32),`source` varchar(32) DEFAULT "order",`last_reset_time` bigint DEFAULT 0,`next_reset_time` bigint DEFAULT 0,`upgrade_group` varchar(64) DEFAULT "",`prev_user_group` varchar(64) DEFAULT "",`downgrade_group` varchar(64) DEFAULT "",`allow_wallet_overflow` numeric NOT NULL,`created_at` integer,`updated_at` integer,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `users` (`id` integer,`username` text UNIQUE,`password` text NOT NULL,`display_name` text,`role` integer DEFAULT 1,`status` integer DEFAULT 1,`email` text,`normalized_email` text,`github_id` text,`discord_id` text,`oidc_id` text,`wechat_id` text,`telegram_id` text,`access_token` char(32),`quota` bigint DEFAULT 0,`used_quota` bigint DEFAULT 0,`request_count` integer DEFAULT 0,`group` varchar(64) DEFAULT "default",`aff_code` varchar(32),`aff_count` integer DEFAULT 0,`aff_quota` bigint DEFAULT 0,`aff_history` bigint DEFAULT 0,`inviter_id` integer,`deleted_at` datetime,`linux_do_id` text,`setting` text,`remark` varchar(255),`stripe_customer` varchar(64),`created_at` integer,`last_login_at` integer DEFAULT 0,`session_generation` integer NOT NULL DEFAULT 0,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE TABLE `vendors` (`id` integer,`name` text NOT NULL,`name_key` text NOT NULL DEFAULT "",`description` text,`icon` varchar(128),`status` integer DEFAULT 1,`created_time` integer,`updated_time` integer,`deleted_at` datetime,PRIMARY KEY (`id`)); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_abilities_channel_id` ON `abilities`(`channel_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_abilities_priority` ON `abilities`(`priority`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_abilities_tag` ON `abilities`(`tag`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_abilities_weight` ON `abilities`(`weight`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_auth_flow_purpose_expiry` ON `auth_flows`(`purpose`,`expires_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_auth_flows_consumed_at` ON `auth_flows`(`consumed_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_auth_flows_session_id` ON `auth_flows`(`session_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `idx_auth_flows_token_hash` ON `auth_flows`(`token_hash`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_auth_flows_user_id` ON `auth_flows`(`user_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_authz_user_overrides_user_id` ON `authz_user_overrides`(`user_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `idx_billing_log_receipts_operation_key` ON `billing_log_receipts`(`operation_key`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `idx_billing_pre_consume_selections_request_id` ON `billing_pre_consume_selections`(`request_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_billing_settlement_admission` ON `billing_settlements`(`user_id`,`status`,`funding_delta`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_billing_settlement_reconciliation` ON `billing_settlements`(`status`,`funding_delta`,`reconciliation_reviewed_at`,`user_blocking_override`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_billing_settlement_status_funding` ON `billing_settlements`(`status`,`funding_delta`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_billing_settlements_effect_status` ON `billing_settlements`(`effect_status`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_billing_settlements_next_attempt` ON `billing_settlements`(`next_attempt`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `idx_billing_settlements_operation_key` ON `billing_settlements`(`operation_key`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_billing_settlements_pre_consume_request_id` ON `billing_settlements`(`pre_consume_request_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_billing_settlements_status` ON `billing_settlements`(`status`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_billing_settlements_subscription_pre_consume_request_id` ON `billing_settlements`(`subscription_pre_consume_request_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_billing_settlements_updated_at` ON `billing_settlements`(`updated_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `idx_cache_invalidation_target` ON `cache_invalidation_tasks`(`kind`,`entity_key`,`delete_entry`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_cache_invalidation_tasks_next_attempt` ON `cache_invalidation_tasks`(`next_attempt`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_channels_name` ON `channels`(`name`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_channels_tag` ON `channels`(`tag`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_created_at_id` ON `logs`(`created_at`,`id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_created_at_type` ON `logs`(`created_at`,`type`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `idx_custom_oauth_providers_slug` ON `custom_oauth_providers`(`slug`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_logs_channel_id` ON `logs`(`channel_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_logs_group` ON `logs`(`group`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_logs_ip` ON `logs`(`ip`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_logs_is_empty_retry` ON `logs`(`is_empty_retry`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_logs_is_error_retry` ON `logs`(`is_error_retry`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_logs_is_retry` ON `logs`(`is_retry`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_logs_model_name` ON `logs`(`model_name`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_logs_quota` ON `logs`(`quota`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_logs_request_id` ON `logs`(`request_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_logs_token_id` ON `logs`(`token_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_logs_token_name` ON `logs`(`token_name`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_logs_type_quota_created_at` ON `logs`(`type`,`quota`,`created_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_logs_upstream_request_id` ON `logs`(`upstream_request_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_logs_user_id` ON `logs`(`user_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_logs_username` ON `logs`(`username`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `idx_midjourney_billing_claim` ON `midjourney_billing_claims`(`channel_id`,`mj_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_midjourney_billing_claims_billing_request_id` ON `midjourney_billing_claims`(`billing_request_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_midjourney_billing_claims_billing_task_id` ON `midjourney_billing_claims`(`billing_task_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_midjourney_billing_claims_user_id` ON `midjourney_billing_claims`(`user_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_midjourneys_action` ON `midjourneys`(`action`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_midjourneys_finish_time` ON `midjourneys`(`finish_time`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_midjourneys_mj_id` ON `midjourneys`(`mj_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_midjourneys_progress` ON `midjourneys`(`progress`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_midjourneys_start_time` ON `midjourneys`(`start_time`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_midjourneys_status` ON `midjourneys`(`status`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_midjourneys_submit_time` ON `midjourneys`(`submit_time`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_midjourneys_user_id` ON `midjourneys`(`user_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_models_deleted_at` ON `models`(`deleted_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_models_vendor_id` ON `models`(`vendor_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `idx_passkey_credentials_credential_id` ON `passkey_credentials`(`credential_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_passkey_credentials_deleted_at` ON `passkey_credentials`(`deleted_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `idx_passkey_credentials_user_id` ON `passkey_credentials`(`user_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_perf_bucket_ts` ON `perf_metrics`(`bucket_ts`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_perf_metric_flush_receipts_created_at` ON `perf_metric_flush_receipts`(`created_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `idx_perf_metric_flush_receipts_receipt_key` ON `perf_metric_flush_receipts`(`receipt_key`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `idx_perf_model_group_bucket` ON `perf_metrics`(`model_name`,`group`,`bucket_ts`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_prefill_groups_deleted_at` ON `prefill_groups`(`deleted_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_prefill_groups_type` ON `prefill_groups`(`type`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_qdt_created_at` ON `quota_data`(`created_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_qdt_model_user_name` ON `quota_data`(`model_name`,`username`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_quota_data_channel_id` ON `quota_data`(`channel_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_quota_data_node_name` ON `quota_data`(`node_name`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_quota_data_snapshot_retries_retry_until` ON `quota_data_snapshot_retries`(`retry_until`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_quota_data_snapshot_retries_updated_at` ON `quota_data_snapshot_retries`(`updated_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_quota_data_snapshots_created_at` ON `quota_data_snapshots`(`created_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_quota_data_token_id` ON `quota_data`(`token_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_quota_data_use_group` ON `quota_data`(`use_group`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_quota_data_user_id` ON `quota_data`(`user_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_redemptions_deleted_at` ON `redemptions`(`deleted_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `idx_redemptions_key` ON `redemptions`(`key`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_redemptions_name` ON `redemptions`(`name`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_subscription_orders_plan_id` ON `subscription_orders`(`plan_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_subscription_orders_trade_no` ON `subscription_orders`(`trade_no`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_subscription_orders_user_id` ON `subscription_orders`(`user_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `idx_subscription_pre_consume_records_request_id` ON `subscription_pre_consume_records`(`request_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_subscription_pre_consume_records_status` ON `subscription_pre_consume_records`(`status`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_subscription_pre_consume_records_token_id` ON `subscription_pre_consume_records`(`token_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_subscription_pre_consume_records_updated_at` ON `subscription_pre_consume_records`(`updated_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_subscription_pre_consume_records_user_id` ON `subscription_pre_consume_records`(`user_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_subscription_pre_consume_records_user_subscription_id` ON `subscription_pre_consume_records`(`user_subscription_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_system_instances_created_at` ON `system_instances`(`created_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_system_instances_last_seen_at` ON `system_instances`(`last_seen_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_system_instances_started_at` ON `system_instances`(`started_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_system_instances_updated_at` ON `system_instances`(`updated_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `idx_system_tasks_active_key` ON `system_tasks`(`active_key`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_system_tasks_created_at` ON `system_tasks`(`created_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_system_tasks_locked_by` ON `system_tasks`(`locked_by`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_system_tasks_locked_until` ON `system_tasks`(`locked_until`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_system_tasks_status` ON `system_tasks`(`status`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `idx_system_tasks_task_id` ON `system_tasks`(`task_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_system_tasks_type` ON `system_tasks`(`type`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_system_tasks_updated_at` ON `system_tasks`(`updated_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_task_timeout_cursor` ON `tasks`(`submit_time`,`id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_tasks_action` ON `tasks`(`action`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_tasks_channel_id` ON `tasks`(`channel_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_tasks_created_at` ON `tasks`(`created_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_tasks_finish_time` ON `tasks`(`finish_time`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_tasks_platform` ON `tasks`(`platform`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_tasks_progress` ON `tasks`(`progress`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_tasks_start_time` ON `tasks`(`start_time`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_tasks_status` ON `tasks`(`status`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_tasks_submit_time` ON `tasks`(`submit_time`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_tasks_task_id` ON `tasks`(`task_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_tasks_user_id` ON `tasks`(`user_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_tokens_deleted_at` ON `tokens`(`deleted_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `idx_tokens_key` ON `tokens`(`key`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_tokens_name` ON `tokens`(`name`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_tokens_user_id` ON `tokens`(`user_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_top_ups_trade_no` ON `top_ups`(`trade_no`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_top_ups_user_id` ON `top_ups`(`user_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_two_fa_backup_codes_deleted_at` ON `two_fa_backup_codes`(`deleted_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_two_fa_backup_codes_user_id` ON `two_fa_backup_codes`(`user_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_two_fas_deleted_at` ON `two_fas`(`deleted_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_two_fas_user_id` ON `two_fas`(`user_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `idx_user_checkin_date` ON `checkins`(`user_id`,`checkin_date`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_user_id_id` ON `logs`(`user_id`,`id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_user_sub_active` ON `user_subscriptions`(`user_id`,`status`,`end_time`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_user_subscriptions_end_time` ON `user_subscriptions`(`end_time`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_user_subscriptions_next_reset_time` ON `user_subscriptions`(`next_reset_time`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_user_subscriptions_plan_id` ON `user_subscriptions`(`plan_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_user_subscriptions_status` ON `user_subscriptions`(`status`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_user_subscriptions_user_id` ON `user_subscriptions`(`user_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `idx_users_access_token` ON `users`(`access_token`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `idx_users_aff_code` ON `users`(`aff_code`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_users_deleted_at` ON `users`(`deleted_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_users_discord_id` ON `users`(`discord_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_users_display_name` ON `users`(`display_name`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_users_email` ON `users`(`email`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_users_git_hub_id` ON `users`(`github_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_users_inviter_id` ON `users`(`inviter_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_users_linux_do_id` ON `users`(`linux_do_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_users_normalized_email` ON `users`(`normalized_email`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_users_oidc_id` ON `users`(`oidc_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_users_stripe_customer` ON `users`(`stripe_customer`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_users_telegram_id` ON `users`(`telegram_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_users_username` ON `users`(`username`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_users_we_chat_id` ON `users`(`wechat_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `idx_vendors_deleted_at` ON `vendors`(`deleted_at`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE INDEX `index_username_model_name` ON `logs`(`model_name`,`username`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `uk_models_active_name_key` ON `models` (`name_key`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `uk_prefill_name` ON `prefill_groups`(`name`) WHERE deleted_at IS NULL; + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `uk_vendors_active_name_key` ON `vendors` (`name_key`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `ux_authz_user_permission` ON `authz_user_overrides`(`user_id`,`resource`,`action`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `ux_provider_userid` ON `user_oauth_bindings`(`provider_id`,`provider_user_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `ux_user_provider` ON `user_oauth_bindings`(`user_id`,`provider_id`); + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `ux_users_discord_id` ON `users` (`discord_id`) WHERE `discord_id` IS NOT NULL AND `discord_id` <> ''; + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `ux_users_github_id` ON `users` (`github_id`) WHERE `github_id` IS NOT NULL AND `github_id` <> ''; + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `ux_users_linux_do_id` ON `users` (`linux_do_id`) WHERE `linux_do_id` IS NOT NULL AND `linux_do_id` <> ''; + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `ux_users_oidc_id` ON `users` (`oidc_id`) WHERE `oidc_id` IS NOT NULL AND `oidc_id` <> ''; + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `ux_users_telegram_id` ON `users` (`telegram_id`) WHERE `telegram_id` IS NOT NULL AND `telegram_id` <> ''; + +-- MAX_API_SQLITE_SCHEMA_STATEMENT +CREATE UNIQUE INDEX `ux_users_wechat_id` ON `users` (`wechat_id`) WHERE `wechat_id` IS NOT NULL AND `wechat_id` <> ''; From 70b892ac69811386f67cfc7e1647171ac1a2d5e9 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Fri, 16 Oct 2026 02:36:07 +0800 Subject: [PATCH 02/16] feat(ops): add safe sqlite archive rehearsal tooling --- docs/installation/sqlite-operations.md | 56 +++ docs/installation/sqlite-upgrade.md | 4 + tools/sqliteops/main.go | 522 +++++++++++++++++++++++++ tools/sqliteops/main_test.go | 113 ++++++ 4 files changed, 695 insertions(+) create mode 100644 docs/installation/sqlite-operations.md create mode 100644 tools/sqliteops/main.go create mode 100644 tools/sqliteops/main_test.go diff --git a/docs/installation/sqlite-operations.md b/docs/installation/sqlite-operations.md new file mode 100644 index 00000000..cbfed7ea --- /dev/null +++ b/docs/installation/sqlite-operations.md @@ -0,0 +1,56 @@ +# SQLite 运维备份与恢复 / SQLite Operations Backup and Restore + +`tools/sqliteops` 只处理**已停写**的 SQLite 文件副本。它不会打开数据库、执行迁移、复制配置内容或修改业务数据。 + +`tools/sqliteops` operates only on a **quiesced** SQLite file set. It does not open a database, run migrations, copy configuration contents, or modify business data. + +## 成套备份 / Complete archive + +先停止 API 进程、后台写入任务和结算恢复任务，确认没有新的请求或日志写入，然后执行： + +```powershell +go run ./tools/sqliteops backup ` + --quiesced ` + --primary C:\data\ledger.sqlite ` + --log C:\data\logs.sqlite ` + --config C:\data\runtime.env ` + --destination C:\archives\max-api-20261016 +``` + +`--log` 省略或与 `--primary` 相同时，主库同时作为日志库归档。SQLite 的 `-wal` 和 `-shm` 文件存在时会随对应数据库一起归档。配置文件只记录文件名、大小和 SHA-256，**不会复制配置内容或密钥**。 + +The `--log` argument may be omitted, or may point to the primary file when the application uses a shared database. Existing SQLite `-wal` and `-shm` sidecars are archived with their database. Configuration is fingerprinted by name, size, and SHA-256 only; its contents and secrets are never copied. + +## 验证与隔离恢复 / Verify and isolated restore + +归档不得覆盖已有目录。验证归档清单和所有文件哈希： + +```powershell +go run ./tools/sqliteops verify --backup C:\archives\max-api-20261016 +``` + +恢复到全新的空目录： + +```powershell +go run ./tools/sqliteops restore ` + --backup C:\archives\max-api-20261016 ` + --target C:\rehearsals\max-api-20261016 +``` + +验证会拒绝缺失、篡改、路径逃逸和未列入清单的额外文件。恢复只复制已验证的数据库文件和 WAL sidecar；配置仍需人工按安全渠道准备。恢复副本必须先使用当前版本执行初始化、重启和业务快照核对，不能直接接入生产流量。 + +Verification rejects missing, modified, path-escaping, and unlisted files. Restore copies only verified database files and WAL sidecars; configuration must be supplied separately through a secure channel. Run initialization, restart, and business-snapshot checks against the restored copy before any traffic is admitted. + +## 边界 / Boundaries + +- `--quiesced` 是强制门禁；运行中复制单个 SQLite 文件不能证明恢复点一致。 +- 归档不代表主库、日志库和外部支付状态已经形成跨系统原子快照。 +- manual、unknown outcome、跨订阅周期和外部已接受但本地未完成的结算必须单独对账。 +- 新版本写入不保证旧版本可以直接降级；回退使用完整升级前归档，并核对期间新增写入。 +- 本工具不验证 MySQL/PostgreSQL、Redis、多主机、真实渠道、OAuth、掉电恢复或浏览器流程。 + +- `--quiesced` is mandatory; copying one SQLite file while it is live does not establish a consistent restore point. +- An archive is not an atomic snapshot across the business database, log database, and external payment state. +- Manual, unknown-outcome, cross-period, and externally accepted but locally incomplete settlements require separate reconciliation. +- New-version writes are not guaranteed to support direct downgrade; rollback uses the complete pre-upgrade archive and reconciles writes made during the rehearsal. +- This tool does not validate MySQL/PostgreSQL, Redis, multi-host operation, real providers, OAuth, power loss, or browser flows. diff --git a/docs/installation/sqlite-upgrade.md b/docs/installation/sqlite-upgrade.md index 41e8146e..4026de79 100644 --- a/docs/installation/sqlite-upgrade.md +++ b/docs/installation/sqlite-upgrade.md @@ -1,5 +1,9 @@ # SQLite 升级验收 / SQLite upgrade acceptance +备份、清单校验和隔离恢复命令见[SQLite 运维备份与恢复](sqlite-operations.md)。 + +See [SQLite operations backup and restore](sqlite-operations.md) for the archive, manifest verification, and isolated restore commands. + 升级前停止接收新请求，在原数据库完成待处理资金结算及日志副作用，停止服务后一起备份业务库、日志库和配置。SQLite 使用 WAL 时，保留尚未合并的 WAL 文件；运行中的单个数据库文件复制不能作为完整备份。 Before upgrading, stop new traffic, finish pending funding settlements and log effects against their original databases, stop the service, and back up the business database, log database, and configuration together. Preserve uncheckpointed WAL files. Copying only a database file while it is in use does not establish a complete backup. diff --git a/tools/sqliteops/main.go b/tools/sqliteops/main.go new file mode 100644 index 00000000..2e80d623 --- /dev/null +++ b/tools/sqliteops/main.go @@ -0,0 +1,522 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ + +// sqliteops provides a conservative, offline archive for a quiesced SQLite +// business database and its optional independent log database. It never opens +// or mutates either database and never copies configuration contents. +package main + +import ( + "crypto/sha256" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/MAX-API-Next/MAX-API/common" +) + +const ( + manifestName = "manifest.json" + manifestVersion = 1 +) + +type manifest struct { + SchemaVersion int `json:"schema_version"` + CreatedAt string `json:"created_at"` + Tool string `json:"tool"` + Files []manifestFile `json:"files"` + Configuration *configurationHash `json:"configuration,omitempty"` +} + +type manifestFile struct { + Roles []string `json:"roles"` + ArchivePath string `json:"archive_path"` + SourceName string `json:"source_name"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + Required bool `json:"required"` +} + +type configurationHash struct { + SourceName string `json:"source_name"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + Copied bool `json:"copied"` +} + +type sourceFile struct { + roles []string + path string + archivePath string +} + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + + var err error + switch os.Args[1] { + case "backup": + err = runBackup(os.Args[2:]) + case "verify": + err = runVerify(os.Args[2:]) + case "restore": + err = runRestore(os.Args[2:]) + case "help", "-h", "--help": + usage() + return + default: + usage() + err = fmt.Errorf("unknown command %q", os.Args[1]) + } + if err != nil { + fmt.Fprintln(os.Stderr, "sqliteops:", err) + os.Exit(1) + } +} + +func usage() { + fmt.Fprintln(os.Stderr, "usage: sqliteops backup --quiesced --primary PATH --destination DIR [--log PATH] [--config PATH]") + fmt.Fprintln(os.Stderr, " sqliteops verify --backup DIR") + fmt.Fprintln(os.Stderr, " sqliteops restore --backup DIR --target DIR") +} + +func runBackup(args []string) error { + flags := flag.NewFlagSet("backup", flag.ContinueOnError) + flags.SetOutput(io.Discard) + primary := flags.String("primary", "", "business SQLite file") + logPath := flags.String("log", "", "independent log SQLite file") + config := flags.String("config", "", "configuration file to fingerprint, never copied") + destination := flags.String("destination", "", "new archive directory") + quiesced := flags.Bool("quiesced", false, "assert that all application and background writers are stopped") + if err := flags.Parse(args); err != nil { + return err + } + if !*quiesced { + return errors.New("refusing a backup without --quiesced; stop application and background writers first") + } + if strings.TrimSpace(*primary) == "" || strings.TrimSpace(*destination) == "" { + return errors.New("--primary and --destination are required") + } + + root, err := prepareDestination(*destination, *primary, *logPath, *config) + if err != nil { + return err + } + sources, err := collectSources(*primary, *logPath) + if err != nil { + return err + } + m := manifest{ + SchemaVersion: manifestVersion, + CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), + Tool: "MAX-API sqliteops", + } + for _, source := range sources { + entry, err := archiveSource(root, source) + if err != nil { + return err + } + m.Files = append(m.Files, entry) + } + if *config != "" { + fingerprint, err := fingerprintFile(*config) + if err != nil { + return fmt.Errorf("fingerprint config: %w", err) + } + fingerprint.Copied = false + m.Configuration = &fingerprint + } + sort.Slice(m.Files, func(i, j int) bool { return m.Files[i].ArchivePath < m.Files[j].ArchivePath }) + if err := writeManifest(root, m); err != nil { + return err + } + return verifyArchive(root) +} + +func runVerify(args []string) error { + flags := flag.NewFlagSet("verify", flag.ContinueOnError) + flags.SetOutput(io.Discard) + backup := flags.String("backup", "", "archive directory") + if err := flags.Parse(args); err != nil { + return err + } + if strings.TrimSpace(*backup) == "" { + return errors.New("--backup is required") + } + root, err := filepath.Abs(*backup) + if err != nil { + return err + } + return verifyArchive(root) +} + +func runRestore(args []string) error { + flags := flag.NewFlagSet("restore", flag.ContinueOnError) + flags.SetOutput(io.Discard) + backup := flags.String("backup", "", "archive directory") + target := flags.String("target", "", "new empty restore directory") + if err := flags.Parse(args); err != nil { + return err + } + if strings.TrimSpace(*backup) == "" || strings.TrimSpace(*target) == "" { + return errors.New("--backup and --target are required") + } + backupRoot, err := filepath.Abs(*backup) + if err != nil { + return err + } + if err := verifyArchive(backupRoot); err != nil { + return err + } + targetRoot, err := filepath.Abs(*target) + if err != nil { + return err + } + if err := ensureEmptyDestination(targetRoot); err != nil { + return err + } + if pathsOverlap(backupRoot, targetRoot) { + return errors.New("restore target must not overlap the backup archive") + } + if err := os.MkdirAll(targetRoot, 0o700); err != nil { + return err + } + m, err := readManifest(backupRoot) + if err != nil { + return err + } + for _, item := range m.Files { + source, err := safeArchivePath(backupRoot, item.ArchivePath) + if err != nil { + return err + } + destination, err := safeArchivePath(targetRoot, item.ArchivePath) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(destination), 0o700); err != nil { + return err + } + if err := copyRegularFile(source, destination, 0o600); err != nil { + return err + } + } + return copyRegularFile(filepath.Join(backupRoot, manifestName), filepath.Join(targetRoot, manifestName), 0o600) +} + +func collectSources(primary, logPath string) ([]sourceFile, error) { + primaryAbs, err := filepath.Abs(primary) + if err != nil { + return nil, err + } + if err := requireRegularFile(primaryAbs); err != nil { + return nil, fmt.Errorf("primary: %w", err) + } + sources := []sourceFile{{roles: []string{"primary"}, path: primaryAbs, archivePath: filepath.Join("primary", filepath.Base(primaryAbs))}} + if strings.TrimSpace(logPath) == "" || samePath(primaryAbs, logPath) { + sources[0].roles = append(sources[0].roles, "log") + return expandSidecars(sources) + } + logAbs, err := filepath.Abs(logPath) + if err != nil { + return nil, err + } + if err := requireRegularFile(logAbs); err != nil { + return nil, fmt.Errorf("log: %w", err) + } + sources = append(sources, sourceFile{roles: []string{"log"}, path: logAbs, archivePath: filepath.Join("log", filepath.Base(logAbs))}) + return expandSidecars(sources) +} + +func expandSidecars(sources []sourceFile) ([]sourceFile, error) { + var expanded []sourceFile + for _, source := range sources { + expanded = append(expanded, source) + for _, suffix := range []string{"-wal", "-shm"} { + path := source.path + suffix + if _, err := os.Stat(path); err == nil { + if err := requireRegularFile(path); err != nil { + return nil, err + } + expanded = append(expanded, sourceFile{roles: source.roles, path: path, archivePath: source.archivePath + suffix}) + } else if !os.IsNotExist(err) { + return nil, err + } + } + } + return expanded, nil +} + +func archiveSource(root string, source sourceFile) (manifestFile, error) { + destination, err := safeArchivePath(root, source.archivePath) + if err != nil { + return manifestFile{}, err + } + if err := os.MkdirAll(filepath.Dir(destination), 0o700); err != nil { + return manifestFile{}, err + } + if err := copyRegularFile(source.path, destination, 0o600); err != nil { + return manifestFile{}, err + } + fingerprint, err := fingerprintFile(destination) + if err != nil { + return manifestFile{}, err + } + return manifestFile{ + Roles: source.roles, + ArchivePath: filepath.ToSlash(source.archivePath), + SourceName: filepath.Base(source.path), + Size: fingerprint.Size, + SHA256: fingerprint.SHA256, + Required: true, + }, nil +} + +func prepareDestination(destination, primary, logPath, config string) (string, error) { + root, err := filepath.Abs(destination) + if err != nil { + return "", err + } + if err := ensureEmptyDestination(root); err != nil { + return "", err + } + for _, source := range []string{primary, logPath, config} { + if source == "" { + continue + } + sourceAbs, err := filepath.Abs(source) + if err != nil { + return "", err + } + if pathsOverlap(root, filepath.Dir(sourceAbs)) { + return "", errors.New("destination must not be inside a source directory or overlap a source") + } + } + return root, nil +} + +func verifyArchive(root string) error { + m, err := readManifest(root) + if err != nil { + return err + } + seen := map[string]struct{}{} + for _, item := range m.Files { + if item.ArchivePath == "" || item.SHA256 == "" || item.Size < 0 || !item.Required { + return fmt.Errorf("invalid manifest entry %q", item.ArchivePath) + } + if _, exists := seen[item.ArchivePath]; exists { + return fmt.Errorf("duplicate manifest entry %q", item.ArchivePath) + } + seen[item.ArchivePath] = struct{}{} + path, err := safeArchivePath(root, item.ArchivePath) + if err != nil { + return err + } + actual, err := fingerprintFile(path) + if err != nil { + return fmt.Errorf("verify %s: %w", item.ArchivePath, err) + } + if actual.Size != item.Size || actual.SHA256 != item.SHA256 { + return fmt.Errorf("archive file changed: %s", item.ArchivePath) + } + } + return rejectUnlistedFiles(root, seen) +} + +func rejectUnlistedFiles(root string, listed map[string]struct{}) error { + return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + if filepath.ToSlash(rel) == manifestName { + return nil + } + if _, ok := listed[filepath.ToSlash(rel)]; !ok { + return fmt.Errorf("archive contains unlisted file %s", filepath.ToSlash(rel)) + } + return nil + }) +} + +func readManifest(root string) (manifest, error) { + data, err := os.ReadFile(filepath.Join(root, manifestName)) + if err != nil { + return manifest{}, fmt.Errorf("read manifest: %w", err) + } + var result manifest + if err := common.Unmarshal(data, &result); err != nil { + return manifest{}, fmt.Errorf("decode manifest: %w", err) + } + if result.SchemaVersion != manifestVersion || len(result.Files) == 0 { + return manifest{}, errors.New("unsupported or empty archive manifest") + } + return result, nil +} + +func writeManifest(root string, value manifest) error { + data, err := common.Marshal(value) + if err != nil { + return err + } + data = append(data, '\n') + return os.WriteFile(filepath.Join(root, manifestName), data, 0o600) +} + +func fingerprintFile(path string) (configurationHash, error) { + info, err := os.Stat(path) + if err != nil { + return configurationHash{}, err + } + if !info.Mode().IsRegular() { + return configurationHash{}, fmt.Errorf("not a regular file: %s", path) + } + file, err := os.Open(path) + if err != nil { + return configurationHash{}, err + } + defer file.Close() + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + return configurationHash{}, err + } + return configurationHash{SourceName: filepath.Base(path), Size: info.Size(), SHA256: fmt.Sprintf("%x", hash.Sum(nil))}, nil +} + +func copyRegularFile(source, destination string, mode os.FileMode) error { + if err := requireRegularFile(source); err != nil { + return err + } + input, err := os.Open(source) + if err != nil { + return err + } + defer input.Close() + output, err := os.OpenFile(destination, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode) + if err != nil { + return err + } + remove := true + defer func() { + _ = output.Close() + if remove { + _ = os.Remove(destination) + } + }() + if _, err := io.Copy(output, input); err != nil { + return err + } + if err := output.Sync(); err != nil { + return err + } + if err := output.Close(); err != nil { + return err + } + remove = false + return nil +} + +func ensureEmptyDestination(path string) error { + info, err := os.Stat(path) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + if !info.IsDir() { + return fmt.Errorf("destination is not a directory: %s", path) + } + entries, err := os.ReadDir(path) + if err != nil { + return err + } + if len(entries) > 0 { + return fmt.Errorf("destination must be new or empty: %s", path) + } + return nil +} + +func requireRegularFile(path string) error { + info, err := os.Lstat(path) + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return fmt.Errorf("file must be regular and not a symlink: %s", path) + } + return nil +} + +func safeArchivePath(root, relative string) (string, error) { + if filepath.IsAbs(relative) || filepath.Clean(relative) == "." || strings.HasPrefix(filepath.ToSlash(filepath.Clean(relative)), "../") { + return "", fmt.Errorf("unsafe archive path: %s", relative) + } + path := filepath.Join(root, filepath.FromSlash(relative)) + rootAbs, err := filepath.Abs(root) + if err != nil { + return "", err + } + pathAbs, err := filepath.Abs(path) + if err != nil { + return "", err + } + if !isWithin(rootAbs, pathAbs) { + return "", fmt.Errorf("archive path escapes root: %s", relative) + } + return pathAbs, nil +} + +func pathsOverlap(left, right string) bool { + leftAbs, _ := filepath.Abs(left) + rightAbs, _ := filepath.Abs(right) + return isWithin(leftAbs, rightAbs) || isWithin(rightAbs, leftAbs) +} + +func isWithin(root, path string) bool { + root = filepath.Clean(root) + path = filepath.Clean(path) + if samePath(root, path) { + return true + } + separator := string(filepath.Separator) + return strings.HasPrefix(path, root+separator) +} + +func samePath(left, right string) bool { + leftAbs, _ := filepath.Abs(left) + rightAbs, _ := filepath.Abs(right) + return strings.EqualFold(filepath.Clean(leftAbs), filepath.Clean(rightAbs)) +} diff --git a/tools/sqliteops/main_test.go b/tools/sqliteops/main_test.go new file mode 100644 index 00000000..6c6a9480 --- /dev/null +++ b/tools/sqliteops/main_test.go @@ -0,0 +1,113 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ + +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBackupVerifyAndRestoreWithWALAndConfigFingerprint(t *testing.T) { + source := t.TempDir() + primary := filepath.Join(source, "ledger.sqlite") + logPath := filepath.Join(source, "logs.sqlite") + config := filepath.Join(source, "runtime.env") + require.NoError(t, os.WriteFile(primary, []byte("ledger-v1"), 0o600)) + require.NoError(t, os.WriteFile(primary+"-wal", []byte("ledger-wal"), 0o600)) + require.NoError(t, os.WriteFile(logPath, []byte("logs-v1"), 0o600)) + require.NoError(t, os.WriteFile(logPath+"-shm", []byte("logs-shm"), 0o600)) + require.NoError(t, os.WriteFile(config, []byte("SECRET=must-not-be-copied\n"), 0o600)) + + archive := filepath.Join(t.TempDir(), "archive") + root, err := prepareDestination(archive, primary, logPath, config) + require.NoError(t, err) + sources, err := collectSources(primary, logPath) + require.NoError(t, err) + value := manifest{ + SchemaVersion: manifestVersion, + CreatedAt: "synthetic", + Tool: "test", + } + for _, sourceFile := range sources { + entry, entryErr := archiveSource(root, sourceFile) + require.NoError(t, entryErr) + value.Files = append(value.Files, entry) + } + fingerprint, err := fingerprintFile(config) + require.NoError(t, err) + fingerprint.Copied = false + value.Configuration = &fingerprint + require.NoError(t, writeManifest(root, value)) + require.NoError(t, verifyArchive(root)) + + restore := filepath.Join(t.TempDir(), "restore") + require.NoError(t, os.MkdirAll(restore, 0o700)) + require.NoError(t, runRestore([]string{"--backup", root, "--target", restore})) + require.Equal(t, []byte("ledger-v1"), mustRead(t, filepath.Join(restore, "primary", "ledger.sqlite"))) + require.Equal(t, []byte("ledger-wal"), mustRead(t, filepath.Join(restore, "primary", "ledger.sqlite-wal"))) + require.Equal(t, []byte("logs-v1"), mustRead(t, filepath.Join(restore, "log", "logs.sqlite"))) + require.NoFileExists(t, filepath.Join(restore, "runtime.env")) + manifestBytes := mustRead(t, filepath.Join(restore, manifestName)) + require.NotContains(t, string(manifestBytes), "SECRET=must-not-be-copied") +} + +func TestBackupRequiresQuiesced(t *testing.T) { + primary := filepath.Join(t.TempDir(), "ledger.sqlite") + require.NoError(t, os.WriteFile(primary, []byte("ledger"), 0o600)) + err := runBackup([]string{"--primary", primary, "--destination", filepath.Join(t.TempDir(), "archive")}) + require.ErrorContains(t, err, "--quiesced") +} + +func TestVerifyRejectsTamperingAndUnlistedFiles(t *testing.T) { + source := t.TempDir() + primary := filepath.Join(source, "ledger.sqlite") + require.NoError(t, os.WriteFile(primary, []byte("ledger"), 0o600)) + archive := filepath.Join(t.TempDir(), "archive") + require.NoError(t, runBackup([]string{"--quiesced", "--primary", primary, "--destination", archive})) + require.NoError(t, os.WriteFile(filepath.Join(archive, "primary", "ledger.sqlite"), []byte("tampered"), 0o600)) + require.ErrorContains(t, verifyArchive(archive), "archive file changed") + + archive2 := filepath.Join(t.TempDir(), "archive") + require.NoError(t, runBackup([]string{"--quiesced", "--primary", primary, "--destination", archive2})) + require.NoError(t, os.WriteFile(filepath.Join(archive2, "extra.txt"), []byte("mixed"), 0o600)) + require.ErrorContains(t, verifyArchive(archive2), "unlisted file") +} + +func TestBackupRejectsDestinationInsideSource(t *testing.T) { + source := t.TempDir() + primary := filepath.Join(source, "ledger.sqlite") + require.NoError(t, os.WriteFile(primary, []byte("ledger"), 0o600)) + err := runBackup([]string{ + "--quiesced", + "--primary", primary, + "--destination", filepath.Join(source, "archive"), + }) + require.ErrorContains(t, err, "source directory") +} + +func mustRead(t *testing.T, path string) []byte { + t.Helper() + data, err := os.ReadFile(path) + require.NoError(t, err) + return data +} From 14a531fb68bca5882997f1ab30555688940596f5 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Fri, 16 Oct 2026 03:02:48 +0800 Subject: [PATCH 03/16] test(sqlite): rehearse restored business snapshots --- docs/installation/sqlite-operations.md | 12 + model/sqlite_operations_acceptance_test.go | 321 +++++++++++++++++++++ 2 files changed, 333 insertions(+) create mode 100644 model/sqlite_operations_acceptance_test.go diff --git a/docs/installation/sqlite-operations.md b/docs/installation/sqlite-operations.md index cbfed7ea..c14664ac 100644 --- a/docs/installation/sqlite-operations.md +++ b/docs/installation/sqlite-operations.md @@ -41,6 +41,16 @@ go run ./tools/sqliteops restore ` Verification rejects missing, modified, path-escaping, and unlisted files. Restore copies only verified database files and WAL sidecars; configuration must be supplied separately through a secure channel. Run initialization, restart, and business-snapshot checks against the restored copy before any traffic is admitted. +## 业务快照验收 / Business snapshot rehearsal + +仓库中的合成验收会使用当前模型创建主库和独立日志库，执行归档与两次隔离恢复，然后重新打开恢复文件，比较用户额度、Token、渠道用量、订阅周期与预消费、settlement、日志和 receipt。它只验证文件归档及当前 SQLite 模型数据的可读性，不代表真实生产库、跨库原子性或外部支付状态已经验证： + +```powershell +go test ./model -run TestSQLiteOperationsAcceptancePreservesBusinessSnapshot -count=1 +``` + +The repository rehearsal creates a synthetic primary database and independent log database from the current models, archives them, restores them twice into isolated directories, reopens both copies, and compares wallet quota, token and channel usage, subscription/pre-consume state, settlement, logs, and receipts. It validates the archive and current SQLite model readability only; it does not establish production-database, cross-database atomicity, or external-payment correctness. + ## 边界 / Boundaries - `--quiesced` 是强制门禁；运行中复制单个 SQLite 文件不能证明恢复点一致。 @@ -48,9 +58,11 @@ Verification rejects missing, modified, path-escaping, and unlisted files. Resto - manual、unknown outcome、跨订阅周期和外部已接受但本地未完成的结算必须单独对账。 - 新版本写入不保证旧版本可以直接降级；回退使用完整升级前归档，并核对期间新增写入。 - 本工具不验证 MySQL/PostgreSQL、Redis、多主机、真实渠道、OAuth、掉电恢复或浏览器流程。 +- 合成业务快照测试不能替代真实进程启动、真实计费/渠道调用或 MySQL/PostgreSQL 兼容性测试。 - `--quiesced` is mandatory; copying one SQLite file while it is live does not establish a consistent restore point. - An archive is not an atomic snapshot across the business database, log database, and external payment state. - Manual, unknown-outcome, cross-period, and externally accepted but locally incomplete settlements require separate reconciliation. - New-version writes are not guaranteed to support direct downgrade; rollback uses the complete pre-upgrade archive and reconciles writes made during the rehearsal. - This tool does not validate MySQL/PostgreSQL, Redis, multi-host operation, real providers, OAuth, power loss, or browser flows. +- The synthetic business-snapshot test does not replace live-process startup, real billing/provider calls, or MySQL/PostgreSQL compatibility testing. diff --git a/model/sqlite_operations_acceptance_test.go b/model/sqlite_operations_acceptance_test.go new file mode 100644 index 00000000..5f727f4c --- /dev/null +++ b/model/sqlite_operations_acceptance_test.go @@ -0,0 +1,321 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ + +package model + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" + + "github.com/glebarez/sqlite" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +type sqliteOperationsBusinessSnapshot struct { + User struct { + ID int + Quota int64 + UsedQuota int64 + RequestCount int + } + Token struct { + ID int + UserID int + RemainQuota int64 + UsedQuota int64 + } + Channel struct { + ID int + UsedQuota int64 + } + Plan struct { + ID int + TotalAmount int64 + ResetPeriod string + } + Subscription struct { + ID int + UserID int + PlanID int + AmountTotal int64 + AmountUsed int64 + LastResetAt int64 + Status string + } + PreConsume struct { + RequestID string + UserID int + SubscriptionID int + PreConsumed int64 + Status string + } + Selection struct { + RequestID string + Source string + UserID int + EffectiveQuota int64 + } + Settlement struct { + OperationKey string + Source string + UserID int + FundingDelta int64 + AppliedFundingDelta int64 + TokenDelta int64 + AppliedTokenDelta int64 + Status string + EffectStatus string + Revision int64 + } + Log struct { + ID int + UserID int + RequestID string + Quota int + ModelName string + } + Receipt struct { + OperationKey string + ClaimToken string + } +} + +func TestSQLiteOperationsAcceptancePreservesBusinessSnapshot(t *testing.T) { + t.Parallel() + + root := sqliteOperationsRepoRoot(t) + workDir := t.TempDir() + primaryDir := filepath.Join(workDir, "primary") + logDir := filepath.Join(workDir, "log") + configDir := filepath.Join(workDir, "config") + require.NoError(t, os.MkdirAll(primaryDir, 0o700)) + require.NoError(t, os.MkdirAll(logDir, 0o700)) + require.NoError(t, os.MkdirAll(configDir, 0o700)) + primaryPath := filepath.Join(primaryDir, "ledger.sqlite") + logPath := filepath.Join(logDir, "logs.sqlite") + configPath := filepath.Join(configDir, "config.yaml") + require.NoError(t, os.WriteFile(configPath, []byte("database_password: synthetic-only\n"), 0o600)) + + seedSQLiteOperationsDataset(t, primaryPath, logPath) + before := readSQLiteOperationsSnapshot(t, primaryPath, logPath) + + archivePath := filepath.Join(workDir, "archive") + runSQLiteOperationsCLI(t, root, "backup", "--quiesced", "--primary", primaryPath, "--log", logPath, "--config", configPath, "--destination", archivePath) + runSQLiteOperationsCLI(t, root, "verify", "--backup", archivePath) + + firstRestore := filepath.Join(workDir, "restore-first") + secondRestore := filepath.Join(workDir, "restore-rollback") + runSQLiteOperationsCLI(t, root, "restore", "--backup", archivePath, "--target", firstRestore) + runSQLiteOperationsCLI(t, root, "restore", "--backup", archivePath, "--target", secondRestore) + + firstSnapshot := readSQLiteOperationsSnapshot(t, + filepath.Join(firstRestore, "primary", filepath.Base(primaryPath)), + filepath.Join(firstRestore, "log", filepath.Base(logPath)), + ) + // Reopening the restored files models a second cold start and catches + // archives that are readable only while the first connection is open. + secondStartSnapshot := readSQLiteOperationsSnapshot(t, + filepath.Join(firstRestore, "primary", filepath.Base(primaryPath)), + filepath.Join(firstRestore, "log", filepath.Base(logPath)), + ) + rollbackSnapshot := readSQLiteOperationsSnapshot(t, + filepath.Join(secondRestore, "primary", filepath.Base(primaryPath)), + filepath.Join(secondRestore, "log", filepath.Base(logPath)), + ) + + require.Equal(t, before, firstSnapshot, "isolated restore changed the business snapshot") + require.Equal(t, firstSnapshot, secondStartSnapshot, "a repeated cold start changed the business snapshot") + require.Equal(t, firstSnapshot, rollbackSnapshot, "the rollback archive copy diverged from the first restore") + _, err := os.Stat(filepath.Join(firstRestore, filepath.Base(configPath))) + require.ErrorIs(t, err, os.ErrNotExist, "configuration contents must not be restored by sqliteops") +} + +func seedSQLiteOperationsDataset(t *testing.T, primaryPath, logPath string) { + const ( + userID = 91001 + tokenID = 91002 + channelID = 91003 + planID = 91004 + subscriptionID = 91005 + ) + now := int64(1_760_000_000) + requestID := "sqliteops-acceptance-request" + operationKey := "request:sqliteops-acceptance:finalize" + + db := openSQLiteOperationsDatabase(t, primaryPath) + require.NoError(t, db.AutoMigrate( + &User{}, &Token{}, &Channel{}, &SubscriptionPlan{}, &UserSubscription{}, + &SubscriptionPreConsumeRecord{}, &BillingPreConsumeSelection{}, &BillingSettlement{}, + )) + require.NoError(t, db.Create(&User{ + Id: userID, Username: "sqliteops-acceptance", Password: "synthetic-password", + Email: "sqliteops@example.invalid", Quota: 100000, UsedQuota: 2345, + RequestCount: 17, Group: "default", + }).Error) + require.NoError(t, db.Create(&Token{ + Id: tokenID, UserId: userID, Key: "synthetic-token-not-a-secret", Name: "acceptance", + RemainQuota: 87654, UsedQuota: 2346, Status: 1, ExpiredTime: -1, + }).Error) + require.NoError(t, db.Create(&Channel{ + Id: channelID, Type: 1, Key: "synthetic-channel-not-a-secret", Name: "acceptance", + Status: 1, UsedQuota: 4567, Group: "default", + }).Error) + require.NoError(t, db.Create(&SubscriptionPlan{ + Id: planID, Title: "SQLite acceptance plan", Currency: "USD", DurationUnit: "month", + DurationValue: 1, Enabled: true, TotalAmount: 50000, QuotaResetPeriod: SubscriptionResetMonthly, + CreatedAt: now, UpdatedAt: now, + }).Error) + require.NoError(t, db.Create(&UserSubscription{ + Id: subscriptionID, UserId: userID, PlanId: planID, AmountTotal: 50000, + AmountUsed: 1200, StartTime: now - 3600, EndTime: now + 86400, + Status: "active", LastResetTime: now - 3600, NextResetTime: now + 30*86400, + AllowWalletOverflow: true, CreatedAt: now, UpdatedAt: now, + }).Error) + require.NoError(t, db.Create(&SubscriptionPreConsumeRecord{ + RequestId: requestID, UserId: userID, TokenId: tokenID, UserSubscriptionId: subscriptionID, + SubscriptionLastResetTime: now - 3600, ModelName: "synthetic-model", PreConsumed: 300, + Status: "consumed", CreatedAt: now, UpdatedAt: now, + }).Error) + require.NoError(t, db.Create(&BillingPreConsumeSelection{ + RequestID: requestID, Source: BillingSettlementSourceSubscription, UserID: userID, + TokenID: tokenID, ModelName: "synthetic-model", RequestedQuota: 300, + EffectiveQuota: 300, CreatedAt: now, + }).Error) + require.NoError(t, db.Create(&BillingSettlement{ + OperationKey: operationKey, Source: BillingSettlementSourceSubscription, UserID: userID, + SubscriptionID: subscriptionID, TokenID: tokenID, FundingDelta: -300, + AppliedFundingDelta: -300, TokenDelta: 0, AppliedTokenDelta: 0, + Status: BillingSettlementStatusApplied, EffectStatus: BillingSettlementEffectApplied, + Revision: 2, PreConsumeRequestID: requestID, PreConsumeModelName: "synthetic-model", + PreConsumeRequestedQuota: 300, PreConsumeEffectiveQuota: 300, CreatedAt: now, UpdatedAt: now, + }).Error) + require.NoError(t, closeSQLiteOperationsDatabase(db)) + + logDB := openSQLiteOperationsDatabase(t, logPath) + require.NoError(t, logDB.AutoMigrate(&Log{}, &BillingLogReceipt{})) + require.NoError(t, logDB.Create(&Log{ + Id: 91006, UserId: userID, CreatedAt: now, Type: LogTypeConsume, + Username: "sqliteops-acceptance", TokenName: "acceptance", ModelName: "synthetic-model", + Quota: 300, PromptTokens: 10, CompletionTokens: 20, TokenId: tokenID, + ChannelId: channelID, RequestId: requestID, Group: "default", Content: "synthetic log", + }).Error) + require.NoError(t, logDB.Create(&BillingLogReceipt{ + OperationKey: operationKey, ClaimToken: "synthetic-claim-token", CreatedAt: now, + }).Error) + require.NoError(t, closeSQLiteOperationsDatabase(logDB)) +} + +func readSQLiteOperationsSnapshot(t *testing.T, primaryPath, logPath string) sqliteOperationsBusinessSnapshot { + const ( + userID = 91001 + tokenID = 91002 + channelID = 91003 + planID = 91004 + subscriptionID = 91005 + requestID = "sqliteops-acceptance-request" + operationKey = "request:sqliteops-acceptance:finalize" + ) + var snapshot sqliteOperationsBusinessSnapshot + db := openSQLiteOperationsDatabase(t, primaryPath) + var user User + require.NoError(t, db.First(&user, userID).Error) + snapshot.User.ID, snapshot.User.Quota, snapshot.User.UsedQuota, snapshot.User.RequestCount = user.Id, user.Quota, user.UsedQuota, user.RequestCount + var token Token + require.NoError(t, db.First(&token, tokenID).Error) + snapshot.Token.ID, snapshot.Token.UserID, snapshot.Token.RemainQuota, snapshot.Token.UsedQuota = token.Id, token.UserId, token.RemainQuota, token.UsedQuota + var channel Channel + require.NoError(t, db.First(&channel, channelID).Error) + snapshot.Channel.ID, snapshot.Channel.UsedQuota = channel.Id, channel.UsedQuota + var plan SubscriptionPlan + require.NoError(t, db.First(&plan, planID).Error) + snapshot.Plan.ID, snapshot.Plan.TotalAmount, snapshot.Plan.ResetPeriod = plan.Id, plan.TotalAmount, plan.QuotaResetPeriod + var subscription UserSubscription + require.NoError(t, db.First(&subscription, subscriptionID).Error) + snapshot.Subscription.ID, snapshot.Subscription.UserID, snapshot.Subscription.PlanID = subscription.Id, subscription.UserId, subscription.PlanId + snapshot.Subscription.AmountTotal, snapshot.Subscription.AmountUsed = subscription.AmountTotal, subscription.AmountUsed + snapshot.Subscription.LastResetAt, snapshot.Subscription.Status = subscription.LastResetTime, subscription.Status + var preConsume SubscriptionPreConsumeRecord + require.NoError(t, db.Where("request_id = ?", requestID).First(&preConsume).Error) + snapshot.PreConsume.RequestID, snapshot.PreConsume.UserID, snapshot.PreConsume.SubscriptionID = preConsume.RequestId, preConsume.UserId, preConsume.UserSubscriptionId + snapshot.PreConsume.PreConsumed, snapshot.PreConsume.Status = preConsume.PreConsumed, preConsume.Status + var selection BillingPreConsumeSelection + require.NoError(t, db.Where("request_id = ?", requestID).First(&selection).Error) + snapshot.Selection.RequestID, snapshot.Selection.Source, snapshot.Selection.UserID = selection.RequestID, selection.Source, selection.UserID + snapshot.Selection.EffectiveQuota = selection.EffectiveQuota + var settlement BillingSettlement + require.NoError(t, db.Where("operation_key = ?", operationKey).First(&settlement).Error) + snapshot.Settlement.OperationKey, snapshot.Settlement.Source, snapshot.Settlement.UserID = settlement.OperationKey, settlement.Source, settlement.UserID + snapshot.Settlement.FundingDelta, snapshot.Settlement.AppliedFundingDelta = settlement.FundingDelta, settlement.AppliedFundingDelta + snapshot.Settlement.TokenDelta, snapshot.Settlement.AppliedTokenDelta = settlement.TokenDelta, settlement.AppliedTokenDelta + snapshot.Settlement.Status, snapshot.Settlement.EffectStatus, snapshot.Settlement.Revision = settlement.Status, settlement.EffectStatus, settlement.Revision + require.NoError(t, closeSQLiteOperationsDatabase(db)) + + logDB := openSQLiteOperationsDatabase(t, logPath) + var logEntry Log + require.NoError(t, logDB.First(&logEntry, 91006).Error) + snapshot.Log.ID, snapshot.Log.UserID, snapshot.Log.RequestID, snapshot.Log.Quota, snapshot.Log.ModelName = logEntry.Id, logEntry.UserId, logEntry.RequestId, logEntry.Quota, logEntry.ModelName + var receipt BillingLogReceipt + require.NoError(t, logDB.Where("operation_key = ?", operationKey).First(&receipt).Error) + snapshot.Receipt.OperationKey, snapshot.Receipt.ClaimToken = receipt.OperationKey, receipt.ClaimToken + require.NoError(t, closeSQLiteOperationsDatabase(logDB)) + return snapshot +} + +func openSQLiteOperationsDatabase(t *testing.T, path string) *gorm.DB { + db, err := gorm.Open(sqlite.Open(path), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + require.NoError(t, err) + return db +} + +func closeSQLiteOperationsDatabase(db *gorm.DB) error { + sqlDB, err := db.DB() + if err != nil { + return err + } + return sqlDB.Close() +} + +func sqliteOperationsRepoRoot(t *testing.T) string { + _, filename, _, ok := runtime.Caller(0) + require.True(t, ok) + directory := filepath.Dir(filename) + for { + if _, err := os.Stat(filepath.Join(directory, "go.mod")); err == nil { + return directory + } + parent := filepath.Dir(directory) + if parent == directory { + t.Fatalf("could not find repository root from %s", filename) + } + directory = parent + } +} + +func runSQLiteOperationsCLI(t *testing.T, root string, args ...string) { + commandArgs := append([]string{"run", "./tools/sqliteops"}, args...) + command := exec.Command("go", commandArgs...) + command.Dir = root + output, err := command.CombinedOutput() + require.NoErrorf(t, err, "sqliteops %v failed: %s", args, output) +} From 46c1482b2f71806d0ee8c589ff4ca25cb421d61d Mon Sep 17 00:00:00 2001 From: CSCITech Date: Fri, 16 Oct 2026 03:45:28 +0800 Subject: [PATCH 04/16] test(mysql): gate startup migration on isolated empty schema --- model/mysql_migration_acceptance_test.go | 103 +++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 model/mysql_migration_acceptance_test.go diff --git a/model/mysql_migration_acceptance_test.go b/model/mysql_migration_acceptance_test.go new file mode 100644 index 00000000..f2b15f0e --- /dev/null +++ b/model/mysql_migration_acceptance_test.go @@ -0,0 +1,103 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ + +package model + +import ( + "context" + "os" + "regexp" + "strings" + "testing" + "time" + + "github.com/MAX-API-Next/MAX-API/common" + mysqldriver "github.com/go-sql-driver/mysql" + "github.com/stretchr/testify/require" + gormmysql "gorm.io/driver/mysql" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +var mysqlMigrationAcceptanceDatabase = regexp.MustCompile(`^maxapi_mysql_acceptance_[a-z0-9_]{1,32}$`) + +// TestMySQLStartupMigrationOnEmptyDatabase is opt-in. It intentionally +// refuses a normal application database: migration acceptance must use a +// fresh, disposable schema so a mistaken DSN cannot alter user data. +func TestMySQLStartupMigrationOnEmptyDatabase(t *testing.T) { + dsn := strings.TrimSpace(testEnv("TEST_MYSQL_MIGRATION_DSN")) + if dsn == "" { + t.Skip("set TEST_MYSQL_MIGRATION_DSN to a dedicated disposable MySQL database") + } + + config, err := mysqldriver.ParseDSN(dsn) + require.NoError(t, err) + if !mysqlMigrationAcceptanceDatabase.MatchString(config.DBName) { + t.Fatalf("refusing MySQL migration target with unsafe database name") + } + if config.Net != "unix" && (config.TLS == nil || config.TLS.InsecureSkipVerify) { + t.Fatalf("refusing MySQL migration target without certificate-validating TLS") + } + + db, err := gorm.Open(gormmysql.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + require.NoError(t, err) + sqlDB, err := db.DB() + require.NoError(t, err) + t.Cleanup(func() { _ = sqlDB.Close() }) + sqlDB.SetMaxOpenConns(4) + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + var relationCount int + require.NoError(t, sqlDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = DATABASE()").Scan(&relationCount)) + if relationCount != 0 { + t.Fatalf("refusing non-empty MySQL migration target (%d tables)", relationCount) + } + + oldDB, oldLogDB := DB, LOG_DB + oldSQLite, oldMySQL, oldPostgres := common.UsingSQLite, common.UsingMySQL, common.UsingPostgreSQL + oldRunner := logRetryMarkerBackfillAsyncRunner + DB, LOG_DB = db, db + common.UsingSQLite, common.UsingMySQL, common.UsingPostgreSQL = false, true, false + logRetryMarkerBackfillAsyncRunner = func(fn func()) { fn() } + initCol() + t.Cleanup(func() { + DB, LOG_DB = oldDB, oldLogDB + common.UsingSQLite, common.UsingMySQL, common.UsingPostgreSQL = oldSQLite, oldMySQL, oldPostgres + logRetryMarkerBackfillAsyncRunner = oldRunner + initCol() + }) + + require.NoError(t, migrateDB(), "first MySQL startup migration") + require.NoError(t, migrateDB(), "second MySQL startup migration") + for _, model := range []any{ + &User{}, &Token{}, &Channel{}, &BillingSettlement{}, &BillingPreConsumeSelection{}, + &SubscriptionPlan{}, &UserSubscription{}, &SubscriptionPreConsumeRecord{}, &Log{}, &BillingLogReceipt{}, + } { + require.True(t, db.Migrator().HasTable(model), "expected migrated table for %T", model) + } + require.True(t, db.Migrator().HasColumn(&User{}, "normalized_email")) + require.True(t, db.Migrator().HasColumn(&UserSubscription{}, "allow_wallet_overflow")) + require.True(t, db.Migrator().HasColumn(&SubscriptionPlan{}, "allow_wallet_overflow")) + require.True(t, db.Migrator().HasIndex(&BillingSettlement{}, "idx_billing_settlement_reconciliation")) +} + +func testEnv(name string) string { + return strings.TrimSpace(os.Getenv(name)) +} From a17d69560edd73993eb0b3c4ac8a4c9a69b12dbe Mon Sep 17 00:00:00 2001 From: CSCITech Date: Fri, 16 Oct 2026 10:14:24 +0800 Subject: [PATCH 05/16] test(mysql): allow explicitly named dedicated schema --- model/mysql_migration_acceptance_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/model/mysql_migration_acceptance_test.go b/model/mysql_migration_acceptance_test.go index f2b15f0e..f4f766b5 100644 --- a/model/mysql_migration_acceptance_test.go +++ b/model/mysql_migration_acceptance_test.go @@ -39,7 +39,9 @@ var mysqlMigrationAcceptanceDatabase = regexp.MustCompile(`^maxapi_mysql_accepta // TestMySQLStartupMigrationOnEmptyDatabase is opt-in. It intentionally // refuses a normal application database: migration acceptance must use a -// fresh, disposable schema so a mistaken DSN cannot alter user data. +// fresh, disposable schema so a mistaken DSN cannot alter user data. A +// non-standard schema name requires an explicit TEST_MYSQL_MIGRATION_DATABASE +// value in the same process and is still required to be empty. func TestMySQLStartupMigrationOnEmptyDatabase(t *testing.T) { dsn := strings.TrimSpace(testEnv("TEST_MYSQL_MIGRATION_DSN")) if dsn == "" { @@ -48,7 +50,8 @@ func TestMySQLStartupMigrationOnEmptyDatabase(t *testing.T) { config, err := mysqldriver.ParseDSN(dsn) require.NoError(t, err) - if !mysqlMigrationAcceptanceDatabase.MatchString(config.DBName) { + allowedDatabase := strings.TrimSpace(os.Getenv("TEST_MYSQL_MIGRATION_DATABASE")) + if !mysqlMigrationAcceptanceDatabase.MatchString(config.DBName) && config.DBName != allowedDatabase { t.Fatalf("refusing MySQL migration target with unsafe database name") } if config.Net != "unix" && (config.TLS == nil || config.TLS.InsecureSkipVerify) { From ed2c18127e4f6a5a7bd9d21426a39bd43ace4829 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Fri, 16 Oct 2026 11:49:38 +0800 Subject: [PATCH 06/16] test(mysql): audit initialized schema compatibility --- model/mysql_migration_acceptance_test.go | 72 ++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 5 deletions(-) diff --git a/model/mysql_migration_acceptance_test.go b/model/mysql_migration_acceptance_test.go index f4f766b5..13d3b89a 100644 --- a/model/mysql_migration_acceptance_test.go +++ b/model/mysql_migration_acceptance_test.go @@ -21,6 +21,12 @@ package model import ( "context" + "crypto/hmac" + "crypto/sha256" + "crypto/tls" + "database/sql" + "encoding/hex" + "errors" "os" "regexp" "strings" @@ -37,6 +43,32 @@ import ( var mysqlMigrationAcceptanceDatabase = regexp.MustCompile(`^maxapi_mysql_acceptance_[a-z0-9_]{1,32}$`) +func TestMySQLSchemaCompatibilityReadOnly(t *testing.T) { + dsn := strings.TrimSpace(os.Getenv("TEST_MYSQL_SCHEMA_AUDIT_DSN")) + if dsn == "" { + t.Skip("set TEST_MYSQL_SCHEMA_AUDIT_DSN to run the read-only MySQL schema audit") + } + config, err := mysqldriver.ParseDSN(dsn) + require.NoError(t, err) + allowedDatabase := strings.TrimSpace(os.Getenv("TEST_MYSQL_SCHEMA_AUDIT_DATABASE")) + if !mysqlMigrationAcceptanceDatabase.MatchString(config.DBName) && config.DBName != allowedDatabase { + t.Fatalf("refusing MySQL schema audit target with unsafe database name") + } + + db, sqlDB := openMySQLAcceptanceDB(t, dsn) + t.Cleanup(func() { _ = sqlDB.Close() }) + for _, model := range []any{ + &User{}, &Token{}, &Channel{}, &Option{}, &BillingSettlement{}, &BillingPreConsumeSelection{}, + &SubscriptionPlan{}, &UserSubscription{}, &SubscriptionPreConsumeRecord{}, &Log{}, &BillingLogReceipt{}, + } { + require.True(t, db.Migrator().HasTable(model), "expected existing table for %T", model) + } + require.True(t, db.Migrator().HasColumn(&User{}, "normalized_email")) + require.True(t, db.Migrator().HasColumn(&UserSubscription{}, "allow_wallet_overflow")) + require.True(t, db.Migrator().HasColumn(&SubscriptionPlan{}, "allow_wallet_overflow")) + require.True(t, db.Migrator().HasIndex(&BillingSettlement{}, "idx_billing_settlement_reconciliation")) +} + // TestMySQLStartupMigrationOnEmptyDatabase is opt-in. It intentionally // refuses a normal application database: migration acceptance must use a // fresh, disposable schema so a mistaken DSN cannot alter user data. A @@ -54,14 +86,11 @@ func TestMySQLStartupMigrationOnEmptyDatabase(t *testing.T) { if !mysqlMigrationAcceptanceDatabase.MatchString(config.DBName) && config.DBName != allowedDatabase { t.Fatalf("refusing MySQL migration target with unsafe database name") } - if config.Net != "unix" && (config.TLS == nil || config.TLS.InsecureSkipVerify) { + if strings.TrimSpace(os.Getenv("TEST_MYSQL_TLS_CERT_SHA256")) == "" && config.Net != "unix" && (config.TLS == nil || config.TLS.InsecureSkipVerify) { t.Fatalf("refusing MySQL migration target without certificate-validating TLS") } - db, err := gorm.Open(gormmysql.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) - require.NoError(t, err) - sqlDB, err := db.DB() - require.NoError(t, err) + db, sqlDB := openMySQLAcceptanceDB(t, dsn) t.Cleanup(func() { _ = sqlDB.Close() }) sqlDB.SetMaxOpenConns(4) @@ -104,3 +133,36 @@ func TestMySQLStartupMigrationOnEmptyDatabase(t *testing.T) { func testEnv(name string) string { return strings.TrimSpace(os.Getenv(name)) } + +func openMySQLAcceptanceDB(t *testing.T, dsn string) (*gorm.DB, *sql.DB) { + t.Helper() + config, err := mysqldriver.ParseDSN(dsn) + require.NoError(t, err) + if fingerprint := strings.TrimSpace(os.Getenv("TEST_MYSQL_TLS_CERT_SHA256")); fingerprint != "" { + expected, err := hex.DecodeString(fingerprint) + require.NoError(t, err) + require.Len(t, expected, sha256.Size) + config.TLS = &tls.Config{ + InsecureSkipVerify: true, + MinVersion: tls.VersionTLS12, + VerifyConnection: func(state tls.ConnectionState) error { + if len(state.PeerCertificates) == 0 { + return errors.New("mysql server returned no certificate") + } + actual := sha256.Sum256(state.PeerCertificates[0].Raw) + if !hmac.Equal(actual[:], expected) { + return errors.New("mysql certificate pin mismatch") + } + return nil + }, + } + } else if config.Net != "unix" && (config.TLS == nil || config.TLS.InsecureSkipVerify) { + t.Fatalf("refusing MySQL target without certificate-validating TLS") + } + connector, err := mysqldriver.NewConnector(config) + require.NoError(t, err) + sqlDB := sql.OpenDB(connector) + db, err := gorm.Open(gormmysql.New(gormmysql.Config{Conn: sqlDB}), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + require.NoError(t, err) + return db, sqlDB +} From 4c853d3a85be113a4ca051f0e1b2f1ebf8ad17a3 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Fri, 16 Oct 2026 13:36:08 +0800 Subject: [PATCH 07/16] test(mysql): pin TLS for external billing contract --- model/task_billing_external_test.go | 30 ++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/model/task_billing_external_test.go b/model/task_billing_external_test.go index cf9076e8..324bb5de 100644 --- a/model/task_billing_external_test.go +++ b/model/task_billing_external_test.go @@ -2,7 +2,11 @@ package model import ( "context" + "crypto/hmac" + "crypto/sha256" + "crypto/tls" "database/sql" + "encoding/hex" "errors" "flag" "fmt" @@ -86,9 +90,33 @@ func openTaskBillingTestSQL(dialect, dsn string) (*sql.DB, string, error) { // plaintext network connection. Unix sockets are local-only; every // other network type (including tcp4/tcp6) requires certificate- // validating TLS and must not permit plaintext fallback. - if config.Net != "unix" && (config.TLS == nil || config.TLS.InsecureSkipVerify || config.AllowFallbackToPlaintext) { + if config.Net != "unix" && config.AllowFallbackToPlaintext { return nil, "", errTaskBillingUnsafeDatabase } + if config.Net != "unix" { + if fingerprint := strings.TrimSpace(os.Getenv("TEST_MYSQL_TLS_CERT_SHA256")); fingerprint != "" { + expected, err := hex.DecodeString(fingerprint) + if err != nil || len(expected) != sha256.Size { + return nil, "", errTaskBillingUnsafeDatabase + } + config.TLS = &tls.Config{ + InsecureSkipVerify: true, + MinVersion: tls.VersionTLS12, + VerifyConnection: func(state tls.ConnectionState) error { + if len(state.PeerCertificates) == 0 { + return errors.New("mysql server returned no certificate") + } + actual := sha256.Sum256(state.PeerCertificates[0].Raw) + if !hmac.Equal(actual[:], expected) { + return errors.New("mysql certificate pin mismatch") + } + return nil + }, + } + } else if config.TLS == nil || config.TLS.InsecureSkipVerify { + return nil, "", errTaskBillingUnsafeDatabase + } + } config.Timeout, config.ReadTimeout, config.WriteTimeout = 5*time.Second, 10*time.Second, 10*time.Second connector, err := mysqldriver.NewConnector(config) if err != nil { From 1c4f9ff0304faeb347aff5d1e1b7ff8e2591ae14 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Fri, 16 Oct 2026 14:04:22 +0800 Subject: [PATCH 08/16] feat(mysql): support pinned TLS certificates --- model/main.go | 55 +++++++++++++++++++++++++++++++++++ model/mysql_tls_test.go | 63 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 model/mysql_tls_test.go diff --git a/model/main.go b/model/main.go index d06027ad..3e030ad6 100644 --- a/model/main.go +++ b/model/main.go @@ -2,10 +2,15 @@ package model import ( "context" + "crypto/hmac" + "crypto/sha256" + "crypto/tls" "database/sql" + "encoding/hex" "errors" "fmt" "log" + "net/url" "os" "strings" "sync" @@ -16,6 +21,7 @@ import ( "github.com/bytedance/gopkg/util/gopool" "github.com/glebarez/sqlite" + mysqldriver "github.com/go-sql-driver/mysql" "gorm.io/driver/mysql" "gorm.io/driver/postgres" "gorm.io/gorm" @@ -219,6 +225,11 @@ func chooseDB(envName string, isLog bool) (*gorm.DB, error) { } // Use MySQL common.SysLog("using MySQL as database") + pinnedDSN, err := mysqlDSNWithPinnedCertificate(dsn) + if err != nil { + return nil, err + } + dsn = pinnedDSN // check parseTime if !strings.Contains(dsn, "parseTime") { if strings.Contains(dsn, "?") { @@ -248,6 +259,50 @@ func chooseDB(envName string, isLog bool) (*gorm.DB, error) { }) } +var mysqlPinnedTLSConfigs sync.Map + +func mysqlDSNWithPinnedCertificate(dsn string) (string, error) { + fingerprint := strings.TrimSpace(os.Getenv("MYSQL_TLS_CERT_SHA256")) + if fingerprint == "" { + return dsn, nil + } + expected, err := hex.DecodeString(fingerprint) + if err != nil || len(expected) != sha256.Size { + return "", errors.New("MYSQL_TLS_CERT_SHA256 must be a 64-character SHA-256 fingerprint") + } + configName := "max-api-mysql-pin-" + hex.EncodeToString(expected[:8]) + if _, loaded := mysqlPinnedTLSConfigs.LoadOrStore(configName, struct{}{}); !loaded { + config := &tls.Config{ + InsecureSkipVerify: true, + MinVersion: tls.VersionTLS12, + VerifyConnection: func(state tls.ConnectionState) error { + if len(state.PeerCertificates) == 0 { + return errors.New("mysql server returned no certificate") + } + actual := sha256.Sum256(state.PeerCertificates[0].Raw) + if !hmac.Equal(actual[:], expected) { + return errors.New("mysql certificate pin mismatch") + } + return nil + }, + } + if err := mysqldriver.RegisterTLSConfig(configName, config); err != nil { + mysqlPinnedTLSConfigs.Delete(configName) + return "", fmt.Errorf("register MySQL certificate pin: %w", err) + } + } + base, rawQuery, hasQuery := strings.Cut(dsn, "?") + values, err := url.ParseQuery(rawQuery) + if err != nil { + return "", fmt.Errorf("parse MySQL DSN query: %w", err) + } + values.Set("tls", configName) + if !hasQuery { + return base + "?" + values.Encode(), nil + } + return base + "?" + values.Encode(), nil +} + func InitDB() (err error) { db, err := chooseDB("SQL_DSN", false) if err == nil { diff --git a/model/mysql_tls_test.go b/model/mysql_tls_test.go new file mode 100644 index 00000000..bd189d2c --- /dev/null +++ b/model/mysql_tls_test.go @@ -0,0 +1,63 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ + +package model + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMySQLDSNWithPinnedCertificateLeavesDefaultUnchanged(t *testing.T) { + t.Setenv("MYSQL_TLS_CERT_SHA256", "") + dsn := "user:password@tcp(127.0.0.1:3306)/database?parseTime=true" + got, err := mysqlDSNWithPinnedCertificate(dsn) + require.NoError(t, err) + require.Equal(t, dsn, got) +} + +func TestMySQLDSNWithPinnedCertificateRejectsInvalidFingerprint(t *testing.T) { + t.Setenv("MYSQL_TLS_CERT_SHA256", "not-a-fingerprint") + _, err := mysqlDSNWithPinnedCertificate("user:password@tcp(127.0.0.1:3306)/database") + require.EqualError(t, err, "MYSQL_TLS_CERT_SHA256 must be a 64-character SHA-256 fingerprint") +} + +func TestMySQLDSNWithPinnedCertificateAddsReusableTLSConfig(t *testing.T) { + fingerprint := strings.Repeat("a", 64) + t.Setenv("MYSQL_TLS_CERT_SHA256", fingerprint) + dsn := "user:password@tcp(127.0.0.1:3306)/database?parseTime=true" + got, err := mysqlDSNWithPinnedCertificate(dsn) + require.NoError(t, err) + require.Contains(t, got, "parseTime=true") + require.Contains(t, got, "tls=max-api-mysql-pin-aaaaaaaaaaaaaaaa") + + reused, err := mysqlDSNWithPinnedCertificate(dsn) + require.NoError(t, err) + require.Equal(t, got, reused) +} + +func TestMySQLDSNWithPinnedCertificateReplacesUnpinnedTLSOption(t *testing.T) { + t.Setenv("MYSQL_TLS_CERT_SHA256", strings.Repeat("b", 64)) + got, err := mysqlDSNWithPinnedCertificate("user:password@tcp(127.0.0.1:3306)/database?tls=skip-verify&parseTime=true") + require.NoError(t, err) + require.Contains(t, got, "tls=max-api-mysql-pin-bbbbbbbbbbbbbbbb") + require.NotContains(t, got, "tls=skip-verify") +} From 352087045672f5298b76eb5fa6fb5959167d2f26 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Fri, 16 Oct 2026 19:40:48 +0800 Subject: [PATCH 09/16] test(mysql): allow explicitly authorized billing schema --- model/task_billing_external_test.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/model/task_billing_external_test.go b/model/task_billing_external_test.go index 324bb5de..15035e4e 100644 --- a/model/task_billing_external_test.go +++ b/model/task_billing_external_test.go @@ -68,6 +68,8 @@ func taskBillingTestCheck(t *testing.T, err error, stage string) { } // Each external invocation needs a freshly provisioned disposable database. +// A non-standard name is accepted only when the same process explicitly sets +// TEST_TASK_BILLING_MYSQL_DATABASE to that exact database name. // Nothing is dropped: successful and partial migrations are retained for audit. // The unique ownership table rejects a second harness targeting the same DB. type taskBillingTestOwnership struct { @@ -83,7 +85,8 @@ func openTaskBillingTestSQL(dialect, dsn string) (*sql.DB, string, error) { if err != nil { return nil, "", err } - if !taskBillingTestDatabaseName.MatchString(config.DBName) { + allowedDatabase := strings.TrimSpace(os.Getenv("TEST_TASK_BILLING_MYSQL_DATABASE")) + if !taskBillingTestDatabaseName.MatchString(config.DBName) && config.DBName != allowedDatabase { return nil, "", errTaskBillingUnsafeDatabase } // External contract tests may never put database credentials on a @@ -168,6 +171,21 @@ func TestOpenTaskBillingTestSQLRejectsPlaintextNetworkVariants(t *testing.T) { } } +func TestOpenTaskBillingTestSQLAllowsExplicitDedicatedDatabase(t *testing.T) { + t.Setenv("TEST_TASK_BILLING_MYSQL_DATABASE", "test3") + t.Setenv("TEST_MYSQL_TLS_CERT_SHA256", strings.Repeat("0", 64)) + db, name, err := openTaskBillingTestSQL("mysql", "user:password@tcp(127.0.0.1:3306)/test3") + if err != nil { + t.Fatalf("expected explicitly authorized database name to pass validation: %v", err) + } + if name != "test3" { + t.Fatalf("expected database name test3, got %q", name) + } + if err := db.Close(); err != nil { + t.Fatalf("close validated database handle: %v", err) + } +} + func taskBillingPostgresSSLMode(dsn string) string { if parsed, err := url.Parse(dsn); err == nil && parsed.Scheme != "" { return strings.ToLower(strings.TrimSpace(parsed.Query().Get("sslmode"))) From 9c5aa92f281f7fdbfbf08ac06e47416ec8f259ae Mon Sep 17 00:00:00 2001 From: CSCITech Date: Fri, 16 Oct 2026 19:52:50 +0800 Subject: [PATCH 10/16] feat(taskplugin): validate plugin manifest contract --- pkg/taskplugin/manifest.go | 327 ++++++++++++++++++++++++++++++++ pkg/taskplugin/manifest_test.go | 99 ++++++++++ 2 files changed, 426 insertions(+) create mode 100644 pkg/taskplugin/manifest.go create mode 100644 pkg/taskplugin/manifest_test.go diff --git a/pkg/taskplugin/manifest.go b/pkg/taskplugin/manifest.go new file mode 100644 index 00000000..1d716e27 --- /dev/null +++ b/pkg/taskplugin/manifest.go @@ -0,0 +1,327 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ + +package taskplugin + +import ( + "fmt" + "net/url" + "regexp" + "strings" + "unicode" + "unicode/utf8" +) + +const APIVersion = 1 + +var ( + manifestKeyPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{0,29}$`) + semverPattern = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$`) + identifierPattern = regexp.MustCompile(`^[A-Za-z_$][A-Za-z0-9_$]*$`) + allowedRouteMethods = map[string]struct{}{"GET": {}, "POST": {}, "PUT": {}, "PATCH": {}, "DELETE": {}} + allowedRouteTypes = map[string]struct{}{"submit": {}, "query": {}, "dynamic": {}} + allowedProtocolNames = map[string]struct{}{"openai_responses": {}, "openai_video": {}, "openai_image": {}} + allowedProtocolModes = map[string]struct{}{"stream": {}, "sync": {}, "background": {}} + allowedUsageTypes = map[string]struct{}{"number": {}, "boolean": {}, "enum": {}} + allowedUsageUnits = map[string]struct{}{"second": {}, "count": {}, "token": {}, "credit": {}} +) + +// Manifest is the host-owned portion of the Task Plugin v1 contract. It is +// deliberately data-only: executing plugin code belongs to a later sandboxed +// driver layer and is not part of validation or registry publication. +type Manifest struct { + APIVersion int + Key string + Name string + Version string + Models []string + Protocols []ProtocolClaim + Routes []Route + BaseURL string + AllowedHosts []string + Upstreams []string + UsageSchema map[string]UsageField +} + +type ProtocolClaim struct { + Name string + Supports []string + Bare bool +} + +type Route struct { + Method string + Path string + Type string + Decode string + Render string + TaskIDParam string + Models []string + RetainResult *bool +} + +type UsageField struct { + Type string + Unit string + Enum []string +} + +func ValidateManifest(manifest Manifest) error { + if manifest.APIVersion != APIVersion { + return fmt.Errorf("unsupported task plugin api version %d", manifest.APIVersion) + } + if !manifestKeyPattern.MatchString(manifest.Key) { + return fmt.Errorf("invalid task plugin key %q", manifest.Key) + } + if strings.TrimSpace(manifest.Name) == "" { + return fmt.Errorf("task plugin name is required") + } + if !semverPattern.MatchString(manifest.Version) { + return fmt.Errorf("invalid task plugin version %q", manifest.Version) + } + modelSet, err := validateModels(manifest.Models) + if err != nil { + return err + } + if err := validateProtocols(manifest.Protocols); err != nil { + return err + } + if err := validateRoutes(manifest.Routes, modelSet); err != nil { + return err + } + if err := validateHTTPBaseURL(manifest.BaseURL); err != nil { + return err + } + if err := validateAllowedHosts(manifest.AllowedHosts); err != nil { + return err + } + if err := validateUpstreams(manifest.Upstreams); err != nil { + return err + } + return validateUsageSchema(manifest.UsageSchema) +} + +func validateModels(models []string) (map[string]struct{}, error) { + if len(models) == 0 { + return nil, fmt.Errorf("task plugin must declare at least one model") + } + set := make(map[string]struct{}, len(models)) + for _, model := range models { + model = strings.TrimSpace(model) + if model == "" { + return nil, fmt.Errorf("task plugin model must not be empty") + } + if _, exists := set[model]; exists { + return nil, fmt.Errorf("duplicate task plugin model %q", model) + } + set[model] = struct{}{} + } + return set, nil +} + +func validateProtocols(protocols []ProtocolClaim) error { + seen := make(map[string]struct{}, len(protocols)) + for _, protocol := range protocols { + if _, ok := allowedProtocolNames[protocol.Name]; !ok { + return fmt.Errorf("unsupported task plugin protocol %q", protocol.Name) + } + if _, exists := seen[protocol.Name]; exists { + return fmt.Errorf("duplicate task plugin protocol %q", protocol.Name) + } + seen[protocol.Name] = struct{}{} + if protocol.Bare { + if protocol.Name == "openai_responses" || len(protocol.Supports) > 0 { + return fmt.Errorf("bare task plugin protocol %q cannot declare modes", protocol.Name) + } + continue + } + if protocol.Name == "openai_responses" && len(protocol.Supports) == 0 { + return fmt.Errorf("task plugin protocol %q requires supports", protocol.Name) + } + modes := make(map[string]struct{}, len(protocol.Supports)) + for _, mode := range protocol.Supports { + if _, ok := allowedProtocolModes[mode]; !ok { + return fmt.Errorf("unsupported task plugin protocol mode %q", mode) + } + if _, exists := modes[mode]; exists { + return fmt.Errorf("duplicate task plugin protocol mode %q", mode) + } + modes[mode] = struct{}{} + } + } + return nil +} + +func validateRoutes(routes []Route, models map[string]struct{}) error { + seen := make(map[string]struct{}, len(routes)) + for _, route := range routes { + method := strings.ToUpper(strings.TrimSpace(route.Method)) + if _, ok := allowedRouteMethods[method]; !ok { + return fmt.Errorf("unsupported task plugin route method %q", route.Method) + } + if _, ok := allowedRouteTypes[route.Type]; !ok { + return fmt.Errorf("unsupported task plugin route type %q", route.Type) + } + shape, err := routeShape(route.Path) + if err != nil { + return err + } + conflictKey := method + " " + shape + if _, exists := seen[conflictKey]; exists { + return fmt.Errorf("duplicate task plugin route %q", conflictKey) + } + seen[conflictKey] = struct{}{} + if route.Type == "query" { + if route.Decode != "" { + return fmt.Errorf("query route %q must not declare decode", route.Path) + } + if !identifierPattern.MatchString(route.Render) { + return fmt.Errorf("query route %q requires a valid render hook", route.Path) + } + } else { + if !identifierPattern.MatchString(route.Decode) || !identifierPattern.MatchString(route.Render) { + return fmt.Errorf("route %q requires valid decode and render hooks", route.Path) + } + } + routeModels := make(map[string]struct{}, len(route.Models)) + for _, model := range route.Models { + if _, ok := models[model]; !ok { + return fmt.Errorf("route %q references undeclared model %q", route.Path, model) + } + if _, exists := routeModels[model]; exists { + return fmt.Errorf("route %q declares duplicate model %q", route.Path, model) + } + routeModels[model] = struct{}{} + } + } + return nil +} + +func routeShape(rawPath string) (string, error) { + if rawPath == "" || !strings.HasPrefix(rawPath, "/") || strings.ContainsAny(rawPath, "?#") { + return "", fmt.Errorf("invalid task plugin route path %q", rawPath) + } + parsed, err := url.ParseRequestURI(rawPath) + if err != nil || parsed.Path != rawPath || strings.Contains(rawPath, "//") { + return "", fmt.Errorf("invalid task plugin route path %q", rawPath) + } + parts := strings.Split(rawPath, "/") + for index, part := range parts { + if strings.HasPrefix(part, ":") { + if len(part) == 1 || !identifierPattern.MatchString(part[1:]) { + return "", fmt.Errorf("invalid task plugin route parameter in %q", rawPath) + } + parts[index] = ":param" + } else if strings.Contains(part, "*") { + return "", fmt.Errorf("wildcard task plugin route is not allowed in %q", rawPath) + } + } + return strings.Join(parts, "/"), nil +} + +func validateHTTPBaseURL(rawURL string) error { + if rawURL == "" { + return nil + } + parsed, err := url.Parse(rawURL) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" || parsed.Path != "" { + return fmt.Errorf("invalid task plugin base url") + } + if !isASCII(parsed.Host) { + return fmt.Errorf("task plugin base url host must be ascii") + } + return nil +} + +func validateAllowedHosts(hosts []string) error { + seen := make(map[string]struct{}, len(hosts)) + for _, host := range hosts { + if host == "" || strings.ContainsAny(host, "\\/ ") || strings.IndexFunc(host, func(r rune) bool { return unicode.IsControl(r) || unicode.IsSpace(r) }) >= 0 { + return fmt.Errorf("invalid task plugin allowed host %q", host) + } + parsed, err := url.Parse("//" + host) + if err != nil || parsed.Host != host || parsed.Hostname() == "" || parsed.User != nil || parsed.Path != "" || parsed.RawQuery != "" || parsed.Fragment != "" { + return fmt.Errorf("invalid task plugin allowed host %q", host) + } + if _, exists := seen[strings.ToLower(host)]; exists { + return fmt.Errorf("duplicate task plugin allowed host %q", host) + } + seen[strings.ToLower(host)] = struct{}{} + } + return nil +} + +func validateUpstreams(upstreams []string) error { + seen := make(map[string]struct{}, len(upstreams)) + for _, upstream := range upstreams { + if upstream != "vendor" && upstream != "new_api" { + return fmt.Errorf("unsupported task plugin upstream %q", upstream) + } + if _, exists := seen[upstream]; exists { + return fmt.Errorf("duplicate task plugin upstream %q", upstream) + } + seen[upstream] = struct{}{} + } + return nil +} + +func validateUsageSchema(schema map[string]UsageField) error { + for name, field := range schema { + if strings.TrimSpace(name) == "" { + return fmt.Errorf("task plugin usage field name must not be empty") + } + if _, ok := allowedUsageTypes[field.Type]; !ok { + return fmt.Errorf("unsupported task plugin usage field type %q", field.Type) + } + if field.Type == "number" { + if _, ok := allowedUsageUnits[field.Unit]; !ok { + return fmt.Errorf("unsupported task plugin usage unit %q", field.Unit) + } + if len(field.Enum) > 0 { + return fmt.Errorf("task plugin numeric usage field %q must not declare enum values", name) + } + } else if field.Type == "enum" { + if field.Unit != "" || len(field.Enum) == 0 { + return fmt.Errorf("task plugin enum usage field %q requires values and no unit", name) + } + seen := make(map[string]struct{}, len(field.Enum)) + for _, value := range field.Enum { + if strings.TrimSpace(value) == "" { + return fmt.Errorf("task plugin enum usage field %q contains an empty value", name) + } + if _, exists := seen[value]; exists { + return fmt.Errorf("task plugin enum usage field %q contains duplicate value %q", name, value) + } + seen[value] = struct{}{} + } + } else if field.Unit != "" || len(field.Enum) > 0 { + return fmt.Errorf("task plugin usage field %q must not declare a unit or enum values", name) + } + } + return nil +} + +func isASCII(value string) bool { + for _, r := range value { + if r > 127 || !utf8.ValidString(value) { + return false + } + } + return true +} diff --git a/pkg/taskplugin/manifest_test.go b/pkg/taskplugin/manifest_test.go new file mode 100644 index 00000000..4a4daeb7 --- /dev/null +++ b/pkg/taskplugin/manifest_test.go @@ -0,0 +1,99 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ + +package taskplugin + +import ( + "strings" + "testing" +) + +func validManifest() Manifest { + return Manifest{ + APIVersion: 1, + Key: "vendor", + Name: "Vendor", + Version: "1.0.0", + Models: []string{"vendor-model"}, + Protocols: []ProtocolClaim{{Name: "openai_responses", Supports: []string{"stream", "sync", "background"}}, {Name: "openai_video", Bare: true}}, + Routes: []Route{ + {Method: "POST", Path: "/vendor/v1/jobs", Type: "submit", Decode: "createJob", Render: "jobCreated"}, + {Method: "GET", Path: "/vendor/v1/jobs/:task_id", Type: "query", Render: "jobStatus"}, + }, + BaseURL: "https://api.vendor.example", + AllowedHosts: []string{"api.vendor.example:443"}, + Upstreams: []string{"vendor", "new_api"}, + UsageSchema: map[string]UsageField{"duration": {Type: "number", Unit: "second"}, "resolution": {Type: "enum", Enum: []string{"720p", "1080p"}}}, + } +} + +func TestValidateManifestAcceptsVersionedProtocolContract(t *testing.T) { + if err := ValidateManifest(validManifest()); err != nil { + t.Fatalf("valid manifest rejected: %v", err) + } +} + +func TestValidateManifestRejectsRouteShapeConflict(t *testing.T) { + manifest := validManifest() + manifest.Routes = append(manifest.Routes, Route{ + Method: "GET", Path: "/vendor/v1/jobs/:id", Type: "query", Render: "jobStatus", + }) + if err := ValidateManifest(manifest); err == nil || !strings.Contains(err.Error(), "duplicate task plugin route") { + t.Fatalf("expected normalized route conflict, got %v", err) + } +} + +func TestValidateManifestRejectsUnsafeBaseURLAndHost(t *testing.T) { + manifest := validManifest() + manifest.BaseURL = "https://user:password@api.vendor.example" + if err := ValidateManifest(manifest); err == nil { + t.Fatal("expected base URL credentials to be rejected") + } + manifest = validManifest() + manifest.AllowedHosts = []string{"https://api.vendor.example/path"} + if err := ValidateManifest(manifest); err == nil { + t.Fatal("expected allowed-host URL form to be rejected") + } +} + +func TestValidateManifestRejectsInvalidProtocolAndUsage(t *testing.T) { + manifest := validManifest() + manifest.Protocols[0].Supports = []string{"stream", "stream"} + if err := ValidateManifest(manifest); err == nil { + t.Fatal("expected duplicate protocol mode to be rejected") + } + manifest = validManifest() + manifest.UsageSchema["duration"] = UsageField{Type: "number", Unit: "usd"} + if err := ValidateManifest(manifest); err == nil { + t.Fatal("expected unsupported usage unit to be rejected") + } + manifest = validManifest() + manifest.UsageSchema["resolution"] = UsageField{Type: "enum", Enum: []string{"720p", "720p"}} + if err := ValidateManifest(manifest); err == nil { + t.Fatal("expected duplicate enum value to be rejected") + } +} + +func TestValidateManifestRejectsUndeclaredRouteModel(t *testing.T) { + manifest := validManifest() + manifest.Routes[0].Models = []string{"missing-model"} + if err := ValidateManifest(manifest); err == nil { + t.Fatal("expected undeclared route model to be rejected") + } +} From 5cc3c116b17f9d8674237418c7bd9c9977735c07 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Fri, 16 Oct 2026 20:07:08 +0800 Subject: [PATCH 11/16] feat(taskplugin): add atomic plugin registry --- pkg/taskplugin/registry.go | 265 ++++++++++++++++++++++++++++++++ pkg/taskplugin/registry_test.go | 120 +++++++++++++++ 2 files changed, 385 insertions(+) create mode 100644 pkg/taskplugin/registry.go create mode 100644 pkg/taskplugin/registry_test.go diff --git a/pkg/taskplugin/registry.go b/pkg/taskplugin/registry.go new file mode 100644 index 00000000..55923d51 --- /dev/null +++ b/pkg/taskplugin/registry.go @@ -0,0 +1,265 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ + +package taskplugin + +import ( + "fmt" + "net/url" + "sort" + "strings" + "sync/atomic" +) + +type Registry struct { + state atomic.Pointer[registryState] +} + +type registryState struct { + generation uint64 + plugins map[string]Manifest + routes map[string]routeBinding + byMethod map[string][]routeBinding + byModel map[string][]string +} + +type Snapshot struct { + state *registryState +} + +type RouteMatch struct { + PluginKey string + Manifest Manifest + Route Route +} + +func NewRegistry() *Registry { + registry := &Registry{} + registry.state.Store(®istryState{ + plugins: map[string]Manifest{}, + routes: map[string]routeBinding{}, + byMethod: map[string][]routeBinding{}, + byModel: map[string][]string{}, + }) + return registry +} + +func (registry *Registry) Publish(manifest Manifest) (uint64, error) { + if registry == nil { + return 0, fmt.Errorf("task plugin registry is nil") + } + if err := ValidateManifest(manifest); err != nil { + return 0, err + } + manifest = cloneManifest(manifest) + for { + current := registry.state.Load() + if _, exists := current.plugins[manifest.Key]; exists { + return 0, fmt.Errorf("task plugin %q is already published", manifest.Key) + } + next := cloneState(current) + next.generation++ + next.plugins[manifest.Key] = manifest + for _, route := range manifest.Routes { + shape, _ := routeShape(route.Path) + method := strings.ToUpper(strings.TrimSpace(route.Method)) + key := method + " " + shape + if existing, exists := current.routes[key]; exists { + return 0, fmt.Errorf("task plugin route %s conflicts with plugin %q", key, existing.pluginKey) + } + binding := routeBinding{pluginKey: manifest.Key, route: cloneRoute(route)} + next.routes[key] = binding + next.byMethod[method] = append(next.byMethod[method], binding) + } + for _, model := range manifest.Models { + next.byModel[model] = append(next.byModel[model], manifest.Key) + sort.Strings(next.byModel[model]) + } + for method := range next.byMethod { + sort.SliceStable(next.byMethod[method], func(i, j int) bool { + return routeSpecificity(next.byMethod[method][i].route.Path) > routeSpecificity(next.byMethod[method][j].route.Path) + }) + } + if registry.state.CompareAndSwap(current, next) { + return next.generation, nil + } + } +} + +func (registry *Registry) Snapshot() Snapshot { + if registry == nil { + return Snapshot{} + } + return Snapshot{state: registry.state.Load()} +} + +func (snapshot Snapshot) Generation() uint64 { + if snapshot.state == nil { + return 0 + } + return snapshot.state.generation +} + +func (snapshot Snapshot) Plugin(key string) (Manifest, bool) { + if snapshot.state == nil { + return Manifest{}, false + } + manifest, ok := snapshot.state.plugins[key] + if !ok { + return Manifest{}, false + } + return cloneManifest(manifest), true +} + +func (snapshot Snapshot) ModelCandidates(model string) []Manifest { + if snapshot.state == nil { + return nil + } + keys := snapshot.state.byModel[model] + result := make([]Manifest, 0, len(keys)) + for _, key := range keys { + if manifest, ok := snapshot.state.plugins[key]; ok { + result = append(result, cloneManifest(manifest)) + } + } + return result +} + +func (snapshot Snapshot) MatchRoute(method, requestPath, model string) (RouteMatch, bool) { + if snapshot.state == nil { + return RouteMatch{}, false + } + method = strings.ToUpper(strings.TrimSpace(method)) + for _, binding := range snapshot.state.byMethod[method] { + if !routeMatches(binding.route.Path, requestPath) || !routeAllowsModel(binding.route, model) { + continue + } + manifest, ok := snapshot.state.plugins[binding.pluginKey] + if !ok { + continue + } + return RouteMatch{PluginKey: binding.pluginKey, Manifest: cloneManifest(manifest), Route: cloneRoute(binding.route)}, true + } + return RouteMatch{}, false +} + +type routeBinding struct { + pluginKey string + route Route +} + +func cloneState(source *registryState) *registryState { + next := ®istryState{ + generation: source.generation, + plugins: make(map[string]Manifest, len(source.plugins)+1), + routes: make(map[string]routeBinding, len(source.routes)+1), + byMethod: make(map[string][]routeBinding, len(source.byMethod)), + byModel: make(map[string][]string, len(source.byModel)), + } + for key, manifest := range source.plugins { + next.plugins[key] = manifest + } + for key, binding := range source.routes { + next.routes[key] = binding + } + for method, bindings := range source.byMethod { + next.byMethod[method] = append([]routeBinding(nil), bindings...) + } + for model, keys := range source.byModel { + next.byModel[model] = append([]string(nil), keys...) + } + return next +} + +func routeAllowsModel(route Route, model string) bool { + if len(route.Models) == 0 { + return true + } + for _, allowed := range route.Models { + if allowed == model { + return true + } + } + return false +} + +func routeMatches(pattern, requestPath string) bool { + parsed, err := url.ParseRequestURI(requestPath) + if err != nil || parsed.Path != requestPath { + return false + } + patternParts := strings.Split(pattern, "/") + requestParts := strings.Split(requestPath, "/") + if len(patternParts) != len(requestParts) { + return false + } + for index, part := range patternParts { + if strings.HasPrefix(part, ":") { + if requestParts[index] == "" { + return false + } + continue + } + if part != requestParts[index] { + return false + } + } + return true +} + +func routeSpecificity(path string) int { + score := 0 + for _, part := range strings.Split(path, "/") { + if part != "" && !strings.HasPrefix(part, ":") { + score++ + } + } + return score +} + +func cloneManifest(manifest Manifest) Manifest { + manifest.Models = append([]string(nil), manifest.Models...) + manifest.Protocols = append([]ProtocolClaim(nil), manifest.Protocols...) + for index := range manifest.Protocols { + manifest.Protocols[index].Supports = append([]string(nil), manifest.Protocols[index].Supports...) + } + manifest.Routes = append([]Route(nil), manifest.Routes...) + for index := range manifest.Routes { + manifest.Routes[index] = cloneRoute(manifest.Routes[index]) + } + manifest.AllowedHosts = append([]string(nil), manifest.AllowedHosts...) + manifest.Upstreams = append([]string(nil), manifest.Upstreams...) + if manifest.UsageSchema != nil { + manifest.UsageSchema = make(map[string]UsageField, len(manifest.UsageSchema)) + for key, field := range manifest.UsageSchema { + field.Enum = append([]string(nil), field.Enum...) + manifest.UsageSchema[key] = field + } + } + return manifest +} + +func cloneRoute(route Route) Route { + route.Models = append([]string(nil), route.Models...) + if route.RetainResult != nil { + value := *route.RetainResult + route.RetainResult = &value + } + return route +} diff --git a/pkg/taskplugin/registry_test.go b/pkg/taskplugin/registry_test.go new file mode 100644 index 00000000..a968afe5 --- /dev/null +++ b/pkg/taskplugin/registry_test.go @@ -0,0 +1,120 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ + +package taskplugin + +import ( + "fmt" + "sync" + "testing" +) + +func registryManifest(key string, route string) Manifest { + manifest := validManifest() + manifest.Key = key + manifest.Name = key + manifest.Routes = []Route{{Method: "GET", Path: route, Type: "query", Render: "jobStatus"}} + return manifest +} + +func TestRegistryPublishesAtomicSnapshotAndMatchesRoute(t *testing.T) { + registry := NewRegistry() + firstGeneration, err := registry.Publish(registryManifest("alpha", "/alpha/jobs/:task_id")) + if err != nil { + t.Fatalf("publish first manifest: %v", err) + } + before := registry.Snapshot() + if before.Generation() != firstGeneration { + t.Fatalf("expected generation %d, got %d", firstGeneration, before.Generation()) + } + if _, ok := before.MatchRoute("GET", "/beta/jobs/1", "vendor-model"); ok { + t.Fatal("unexpected route match before second publication") + } + secondGeneration, err := registry.Publish(registryManifest("beta", "/beta/jobs/:task_id")) + if err != nil { + t.Fatalf("publish second manifest: %v", err) + } + if secondGeneration != firstGeneration+1 { + t.Fatalf("expected generation %d, got %d", firstGeneration+1, secondGeneration) + } + if _, ok := before.Plugin("beta"); ok { + t.Fatal("old snapshot observed a later publication") + } + match, ok := registry.Snapshot().MatchRoute("GET", "/beta/jobs/1", "vendor-model") + if !ok || match.PluginKey != "beta" || match.Route.Path != "/beta/jobs/:task_id" { + t.Fatalf("unexpected route match: %#v, %v", match, ok) + } +} + +func TestRegistryRejectsDuplicatePluginAndRouteConflict(t *testing.T) { + registry := NewRegistry() + if _, err := registry.Publish(registryManifest("alpha", "/jobs/:task_id")); err != nil { + t.Fatalf("publish first manifest: %v", err) + } + if _, err := registry.Publish(registryManifest("alpha", "/other/:id")); err == nil { + t.Fatal("expected duplicate plugin key to be rejected") + } + if _, err := registry.Publish(registryManifest("beta", "/jobs/:id")); err == nil { + t.Fatal("expected normalized route conflict to be rejected") + } +} + +func TestRegistryIndexesModelCandidatesInStableOrder(t *testing.T) { + registry := NewRegistry() + for _, key := range []string{"zeta", "alpha", "middle"} { + manifest := registryManifest(key, "/"+key) + if _, err := registry.Publish(manifest); err != nil { + t.Fatalf("publish %s: %v", key, err) + } + } + candidates := registry.Snapshot().ModelCandidates("vendor-model") + if len(candidates) != 3 || candidates[0].Key != "alpha" || candidates[1].Key != "middle" || candidates[2].Key != "zeta" { + t.Fatalf("unexpected model candidates: %#v", candidates) + } +} + +func TestRegistryConcurrentPublicationPreservesAllUniquePlugins(t *testing.T) { + registry := NewRegistry() + const count = 12 + var group sync.WaitGroup + errors := make(chan error, count) + for index := 0; index < count; index++ { + index := index + group.Add(1) + go func() { + defer group.Done() + key := fmt.Sprintf("p%02d", index) + if _, err := registry.Publish(registryManifest(key, "/"+key)); err != nil { + errors <- err + } + }() + } + group.Wait() + close(errors) + for err := range errors { + t.Fatalf("concurrent publication failed: %v", err) + } + snapshot := registry.Snapshot() + if snapshot.Generation() != count { + t.Fatalf("expected generation %d, got %d", count, snapshot.Generation()) + } + if len(snapshot.ModelCandidates("vendor-model")) != count { + t.Fatalf("expected %d model candidates", count) + } +} From c9b0445931f8b90351c9c8925f54484e9be199ba Mon Sep 17 00:00:00 2001 From: CSCITech Date: Fri, 16 Oct 2026 20:28:06 +0800 Subject: [PATCH 12/16] test(taskplugin): add deterministic mock driver --- pkg/taskplugin/mock.go | 154 ++++++++++++++++++++++++++++ pkg/taskplugin/mock_test.go | 169 +++++++++++++++++++++++++++++++ service/task_plugin_mock_test.go | 59 +++++++++++ 3 files changed, 382 insertions(+) create mode 100644 pkg/taskplugin/mock.go create mode 100644 pkg/taskplugin/mock_test.go create mode 100644 service/task_plugin_mock_test.go diff --git a/pkg/taskplugin/mock.go b/pkg/taskplugin/mock.go new file mode 100644 index 00000000..734ca7e2 --- /dev/null +++ b/pkg/taskplugin/mock.go @@ -0,0 +1,154 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ + +package taskplugin + +import ( + "fmt" + "sync" + + "github.com/MAX-API-Next/MAX-API/pkg/taskusage" + "github.com/MAX-API-Next/MAX-API/types" +) + +const ( + MockTaskStatusInProgress = "in_progress" + MockTaskStatusSuccess = "success" +) + +type MockDriver struct { + mu sync.Mutex + contract types.TaskUsageContract + usage *types.TaskUsage + nextID uint64 + tasks map[string]*mockTask +} + +type MockTaskResult struct { + TaskID string + Status string + UsageEnvelope *types.TaskUsageEnvelope +} + +type mockTask struct { + status string + polls int + usageEnvelope *types.TaskUsageEnvelope +} + +func NewMockDriver(contract types.TaskUsageContract, usage *types.TaskUsage) (*MockDriver, error) { + if _, err := taskusage.ContractDigest(contract); err != nil { + return nil, fmt.Errorf("validate mock task usage contract: %w", err) + } + return &MockDriver{ + contract: cloneUsageContract(contract), + usage: types.CloneTaskUsage(usage), + tasks: make(map[string]*mockTask), + }, nil +} + +func (driver *MockDriver) UsageContract() types.TaskUsageContract { + if driver == nil { + return types.TaskUsageContract{} + } + driver.mu.Lock() + defer driver.mu.Unlock() + return cloneUsageContract(driver.contract) +} + +func (driver *MockDriver) UsageProducerKind() string { + return types.TaskUsageProducerKindTaskPlugin +} + +func (driver *MockDriver) ProduceUsage(context types.TaskUsageContext) (*types.TaskUsageEnvelope, error) { + if driver == nil { + return nil, fmt.Errorf("mock task plugin driver is nil") + } + driver.mu.Lock() + contract := cloneUsageContract(driver.contract) + usage := types.CloneTaskUsage(driver.usage) + driver.mu.Unlock() + if usage != nil { + usage.Source = context.Stage + if usage.Completeness == "" { + usage.Completeness = types.TaskUsageCompletenessComplete + } + } + return taskusage.BuildEnvelope(types.TaskUsageProducerKindTaskPlugin, contract, context.Stage, usage) +} + +func (driver *MockDriver) Submit() (string, error) { + if driver == nil { + return "", fmt.Errorf("mock task plugin driver is nil") + } + driver.mu.Lock() + defer driver.mu.Unlock() + driver.nextID++ + taskID := fmt.Sprintf("mock-task-%d", driver.nextID) + driver.tasks[taskID] = &mockTask{status: MockTaskStatusInProgress} + return taskID, nil +} + +func (driver *MockDriver) Poll(taskID string) (MockTaskResult, error) { + if driver == nil { + return MockTaskResult{}, fmt.Errorf("mock task plugin driver is nil") + } + driver.mu.Lock() + defer driver.mu.Unlock() + task, ok := driver.tasks[taskID] + if !ok { + return MockTaskResult{}, fmt.Errorf("mock task %q not found", taskID) + } + if task.status == MockTaskStatusSuccess { + return MockTaskResult{TaskID: taskID, Status: task.status, UsageEnvelope: types.CloneTaskUsageEnvelope(task.usageEnvelope)}, nil + } + task.polls++ + if task.polls == 1 { + return MockTaskResult{TaskID: taskID, Status: MockTaskStatusInProgress}, nil + } + usage := types.CloneTaskUsage(driver.usage) + if usage != nil { + usage.Source = types.TaskUsageSourceProviderResponse + if usage.Completeness == "" { + usage.Completeness = types.TaskUsageCompletenessComplete + } + } + envelope, err := taskusage.BuildEnvelope(types.TaskUsageProducerKindTaskPlugin, driver.contract, types.TaskUsageSourceProviderResponse, usage) + if err != nil { + return MockTaskResult{}, fmt.Errorf("produce mock task usage: %w", err) + } + task.status = MockTaskStatusSuccess + task.usageEnvelope = types.CloneTaskUsageEnvelope(envelope) + return MockTaskResult{TaskID: taskID, Status: task.status, UsageEnvelope: types.CloneTaskUsageEnvelope(envelope)}, nil +} + +func cloneUsageContract(contract types.TaskUsageContract) types.TaskUsageContract { + contract.Fields = append([]types.TaskUsageFieldContract(nil), contract.Fields...) + for index := range contract.Fields { + if contract.Fields[index].MinValue != nil { + value := *contract.Fields[index].MinValue + contract.Fields[index].MinValue = &value + } + if contract.Fields[index].MaxValue != nil { + value := *contract.Fields[index].MaxValue + contract.Fields[index].MaxValue = &value + } + } + return contract +} diff --git a/pkg/taskplugin/mock_test.go b/pkg/taskplugin/mock_test.go new file mode 100644 index 00000000..fc29ea81 --- /dev/null +++ b/pkg/taskplugin/mock_test.go @@ -0,0 +1,169 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ + +package taskplugin + +import ( + "sync" + "testing" + + "github.com/MAX-API-Next/MAX-API/pkg/taskusage" + "github.com/MAX-API-Next/MAX-API/types" +) + +func TestMockDriverProducesTrustedTaskPluginEnvelope(t *testing.T) { + outputDuration := int64(5_000) + imageCount := int64(0) + driver, err := NewMockDriver(taskusage.MiniMaxH3Contract(), &types.TaskUsage{ + OutputDurationMs: &outputDuration, + InputImageCount: &imageCount, + Completeness: types.TaskUsageCompletenessComplete, + }) + if err != nil { + t.Fatalf("create mock driver: %v", err) + } + taskID, err := driver.Submit() + if err != nil { + t.Fatalf("submit mock task: %v", err) + } + first, err := driver.Poll(taskID) + if err != nil { + t.Fatalf("first mock poll: %v", err) + } + if first.Status != MockTaskStatusInProgress || first.UsageEnvelope != nil { + t.Fatalf("unexpected first poll: %#v", first) + } + terminal, err := driver.Poll(taskID) + if err != nil { + t.Fatalf("terminal mock poll: %v", err) + } + if terminal.Status != MockTaskStatusSuccess || terminal.UsageEnvelope == nil { + t.Fatalf("unexpected terminal poll: %#v", terminal) + } + if err := taskusage.ValidateEnvelope(taskusage.MiniMaxH3Contract(), terminal.UsageEnvelope, types.TaskUsageProducerKindTaskPlugin); err != nil { + t.Fatalf("validate mock envelope: %v", err) + } + if terminal.UsageEnvelope.Presence != types.TaskUsagePresencePresentValid { + t.Fatalf("expected valid usage presence, got %q", terminal.UsageEnvelope.Presence) + } + if terminal.UsageEnvelope.Usage.InputImageCount == nil || *terminal.UsageEnvelope.Usage.InputImageCount != 0 { + t.Fatal("explicit zero image count was not preserved") + } +} + +func TestMockDriverTerminalPollReplayIsStable(t *testing.T) { + outputDuration := int64(4_000) + imageCount := int64(0) + driver, err := NewMockDriver(taskusage.MiniMaxH3Contract(), &types.TaskUsage{ + OutputDurationMs: &outputDuration, + InputImageCount: &imageCount, + Completeness: types.TaskUsageCompletenessComplete, + }) + if err != nil { + t.Fatalf("create mock driver: %v", err) + } + taskID, err := driver.Submit() + if err != nil { + t.Fatalf("submit mock task: %v", err) + } + if _, err := driver.Poll(taskID); err != nil { + t.Fatalf("first mock poll: %v", err) + } + firstTerminal, err := driver.Poll(taskID) + if err != nil { + t.Fatalf("terminal mock poll: %v", err) + } + replay, err := driver.Poll(taskID) + if err != nil { + t.Fatalf("terminal replay: %v", err) + } + if firstTerminal.UsageEnvelope.EvidenceDigest != replay.UsageEnvelope.EvidenceDigest { + t.Fatalf("terminal replay changed evidence digest: %q vs %q", firstTerminal.UsageEnvelope.EvidenceDigest, replay.UsageEnvelope.EvidenceDigest) + } + if firstTerminal.UsageEnvelope.Usage == replay.UsageEnvelope.Usage { + t.Fatal("terminal replay returned mutable usage pointer") + } +} + +func TestMockDriverConcurrentPollsDoNotDuplicateTerminalEvidence(t *testing.T) { + outputDuration := int64(5_000) + imageCount := int64(0) + driver, err := NewMockDriver(taskusage.MiniMaxH3Contract(), &types.TaskUsage{ + OutputDurationMs: &outputDuration, + InputImageCount: &imageCount, + Completeness: types.TaskUsageCompletenessComplete, + }) + if err != nil { + t.Fatalf("create mock driver: %v", err) + } + taskID, err := driver.Submit() + if err != nil { + t.Fatalf("submit mock task: %v", err) + } + var group sync.WaitGroup + results := make(chan MockTaskResult, 8) + errors := make(chan error, 8) + for i := 0; i < 8; i++ { + group.Add(1) + go func() { + defer group.Done() + result, pollErr := driver.Poll(taskID) + if pollErr != nil { + errors <- pollErr + return + } + results <- result + }() + } + group.Wait() + close(results) + close(errors) + for err := range errors { + t.Fatalf("concurrent poll failed: %v", err) + } + terminalDigest := "" + terminalCount := 0 + for result := range results { + if result.Status != MockTaskStatusSuccess { + continue + } + terminalCount++ + if result.UsageEnvelope == nil { + t.Fatal("successful mock poll omitted usage envelope") + } + if terminalDigest == "" { + terminalDigest = result.UsageEnvelope.EvidenceDigest + } else if terminalDigest != result.UsageEnvelope.EvidenceDigest { + t.Fatal("concurrent terminal polls produced different evidence") + } + } + if terminalCount == 0 { + t.Fatal("concurrent polls never reached terminal state") + } +} + +func TestMockDriverRejectsUnknownTask(t *testing.T) { + driver, err := NewMockDriver(taskusage.MiniMaxH3Contract(), nil) + if err != nil { + t.Fatalf("create mock driver: %v", err) + } + if _, err := driver.Poll("missing"); err == nil { + t.Fatal("expected unknown task to be rejected") + } +} diff --git a/service/task_plugin_mock_test.go b/service/task_plugin_mock_test.go new file mode 100644 index 00000000..e3bffd19 --- /dev/null +++ b/service/task_plugin_mock_test.go @@ -0,0 +1,59 @@ +/* +Copyright (C) 2023-2026 MAX-API-Next + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +For commercial licensing, please contact https://github.com/MAX-API-Next/MAX-API/issues +*/ + +package service + +import ( + "testing" + + "github.com/MAX-API-Next/MAX-API/pkg/taskplugin" + "github.com/MAX-API-Next/MAX-API/pkg/taskusage" + relaycommon "github.com/MAX-API-Next/MAX-API/relay/common" + "github.com/MAX-API-Next/MAX-API/types" + "github.com/stretchr/testify/require" +) + +func TestTaskPluginMockDriverEnvelopeAtPollingBoundary(t *testing.T) { + outputDuration := int64(5_000) + imageCount := int64(0) + driver, err := taskplugin.NewMockDriver(taskusage.MiniMaxH3Contract(), &types.TaskUsage{ + OutputDurationMs: &outputDuration, + InputImageCount: &imageCount, + Completeness: types.TaskUsageCompletenessComplete, + }) + require.NoError(t, err) + taskID, err := driver.Submit() + require.NoError(t, err) + _, err = driver.Poll(taskID) + require.NoError(t, err) + terminal, err := driver.Poll(taskID) + require.NoError(t, err) + require.NotNil(t, terminal.UsageEnvelope) + + adaptor := &usageFactPollingResponseAdaptor{ + contract: driver.UsageContract(), + producerKind: driver.UsageProducerKind(), + envelope: terminal.UsageEnvelope, + } + taskResult := &relaycommon.TaskInfo{} + require.NoError(t, applyTaskUsageFacts(adaptor, []byte(`{}`), taskResult)) + require.NotNil(t, taskResult.UsageEnvelope) + require.Equal(t, types.TaskUsageProducerKindTaskPlugin, taskResult.UsageEnvelope.ProducerKind) + require.EqualValues(t, 5_000, *taskResult.Usage.OutputDurationMs) +} From 853ccc919186d944264f761c171991d08351d990 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Fri, 16 Oct 2026 21:34:58 +0800 Subject: [PATCH 13/16] test(taskplugin): exercise sqlite settlement replay --- model/task_billing_contract_test.go | 54 +++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/model/task_billing_contract_test.go b/model/task_billing_contract_test.go index 66225a59..e20b04cc 100644 --- a/model/task_billing_contract_test.go +++ b/model/task_billing_contract_test.go @@ -7,6 +7,8 @@ import ( "time" "github.com/MAX-API-Next/MAX-API/common" + "github.com/MAX-API-Next/MAX-API/pkg/taskplugin" + "github.com/MAX-API-Next/MAX-API/pkg/taskusage" "github.com/MAX-API-Next/MAX-API/types" "github.com/stretchr/testify/require" "gorm.io/gorm" @@ -378,6 +380,58 @@ func runTaskBillingSettlementContracts(t *testing.T, db *gorm.DB) { }) } +func TestMockTaskPluginUsageFlowsThroughSQLiteSettlementReplay(t *testing.T) { + truncateTables(t) + outputDuration := int64(5_000) + imageCount := int64(0) + driver, err := taskplugin.NewMockDriver(taskusage.MiniMaxH3Contract(), &types.TaskUsage{ + OutputDurationMs: &outputDuration, + InputImageCount: &imageCount, + Completeness: types.TaskUsageCompletenessComplete, + }) + require.NoError(t, err) + taskID, err := driver.Submit() + require.NoError(t, err) + _, err = driver.Poll(taskID) + require.NoError(t, err) + terminal, err := driver.Poll(taskID) + require.NoError(t, err) + require.NotNil(t, terminal.UsageEnvelope) + + fixture := newTaskBillingContractFixture(t, DB, 9901, BillingSettlementSourceWallet, -20) + require.NotNil(t, fixture.task.PrivateData.BillingContext) + fixture.task.PrivateData.BillingContext.TaskUsageEnvelope = terminal.UsageEnvelope + require.NoError(t, DB.Save(&fixture.task).Error) + + applied, replay, err := ApplyBillingSettlementOnce(fixture.input) + require.NoError(t, err) + require.EqualValues(t, -20, applied) + require.False(t, replay) + require.NoError(t, ProcessBillingSettlementEffect(fixture.input.OperationKey)) + + _, replay, err = ApplyBillingSettlementOnce(fixture.input) + require.NoError(t, err) + require.True(t, replay) + require.NoError(t, ProcessBillingSettlementEffect(fixture.input.OperationKey)) + fixture.assertProjection(t, DB, 80, 1) + var user User + var token Token + var task Task + require.NoError(t, DB.First(&user, fixture.user.Id).Error) + require.NoError(t, DB.First(&token, fixture.token.Id).Error) + require.NoError(t, DB.First(&task, fixture.task.ID).Error) + require.EqualValues(t, 920, user.Quota) + require.EqualValues(t, 920, token.RemainQuota) + require.EqualValues(t, 80, token.UsedQuota) + require.EqualValues(t, 80, task.Quota) + + var stored Task + require.NoError(t, DB.First(&stored, fixture.task.ID).Error) + require.NotNil(t, stored.PrivateData.BillingContext) + require.NotNil(t, stored.PrivateData.BillingContext.TaskUsageEnvelope) + require.Equal(t, terminal.UsageEnvelope.EvidenceDigest, stored.PrivateData.BillingContext.TaskUsageEnvelope.EvidenceDigest) +} + // Guard parsing is tested without a live server or any user credentials. func TestTaskBillingExternalDatabaseGuard(t *testing.T) { for _, test := range []struct { From 1348246e03ac1044e4b13bad8c8dc6a1c43ffe71 Mon Sep 17 00:00:00 2001 From: CSCITech Date: Sat, 17 Oct 2026 00:04:02 +0800 Subject: [PATCH 14/16] fix: address CodeRabbit task plugin and TLS findings --- model/main.go | 17 ++++++------- model/mysql_migration_acceptance_test.go | 1 + model/mysql_tls_test.go | 15 ++++++++++-- model/task_billing_external_test.go | 1 + pkg/taskplugin/manifest.go | 21 ++++++++++++++-- pkg/taskplugin/manifest_test.go | 21 ++++++++++++++++ pkg/taskplugin/registry_test.go | 1 - tools/sqliteops/main.go | 31 +++++++++++++++++++++--- tools/sqliteops/main_test.go | 8 ++++++ 9 files changed, 97 insertions(+), 19 deletions(-) diff --git a/model/main.go b/model/main.go index 3e030ad6..e60cf140 100644 --- a/model/main.go +++ b/model/main.go @@ -10,7 +10,6 @@ import ( "errors" "fmt" "log" - "net/url" "os" "strings" "sync" @@ -270,9 +269,10 @@ func mysqlDSNWithPinnedCertificate(dsn string) (string, error) { if err != nil || len(expected) != sha256.Size { return "", errors.New("MYSQL_TLS_CERT_SHA256 must be a 64-character SHA-256 fingerprint") } - configName := "max-api-mysql-pin-" + hex.EncodeToString(expected[:8]) + configName := "max-api-mysql-pin-" + hex.EncodeToString(expected) if _, loaded := mysqlPinnedTLSConfigs.LoadOrStore(configName, struct{}{}); !loaded { config := &tls.Config{ + //nolint:gosec // VerifyConnection enforces the configured exact leaf SHA-256 pin. InsecureSkipVerify: true, MinVersion: tls.VersionTLS12, VerifyConnection: func(state tls.ConnectionState) error { @@ -291,16 +291,13 @@ func mysqlDSNWithPinnedCertificate(dsn string) (string, error) { return "", fmt.Errorf("register MySQL certificate pin: %w", err) } } - base, rawQuery, hasQuery := strings.Cut(dsn, "?") - values, err := url.ParseQuery(rawQuery) + cfg, err := mysqldriver.ParseDSN(dsn) if err != nil { - return "", fmt.Errorf("parse MySQL DSN query: %w", err) + return "", fmt.Errorf("parse MySQL DSN: %w", err) } - values.Set("tls", configName) - if !hasQuery { - return base + "?" + values.Encode(), nil - } - return base + "?" + values.Encode(), nil + cfg.TLS = nil + cfg.TLSConfig = configName + return cfg.FormatDSN(), nil } func InitDB() (err error) { diff --git a/model/mysql_migration_acceptance_test.go b/model/mysql_migration_acceptance_test.go index 13d3b89a..8b19f979 100644 --- a/model/mysql_migration_acceptance_test.go +++ b/model/mysql_migration_acceptance_test.go @@ -143,6 +143,7 @@ func openMySQLAcceptanceDB(t *testing.T, dsn string) (*gorm.DB, *sql.DB) { require.NoError(t, err) require.Len(t, expected, sha256.Size) config.TLS = &tls.Config{ + //nolint:gosec // VerifyConnection enforces the configured exact leaf SHA-256 pin. InsecureSkipVerify: true, MinVersion: tls.VersionTLS12, VerifyConnection: func(state tls.ConnectionState) error { diff --git a/model/mysql_tls_test.go b/model/mysql_tls_test.go index bd189d2c..0140f3f0 100644 --- a/model/mysql_tls_test.go +++ b/model/mysql_tls_test.go @@ -47,7 +47,7 @@ func TestMySQLDSNWithPinnedCertificateAddsReusableTLSConfig(t *testing.T) { got, err := mysqlDSNWithPinnedCertificate(dsn) require.NoError(t, err) require.Contains(t, got, "parseTime=true") - require.Contains(t, got, "tls=max-api-mysql-pin-aaaaaaaaaaaaaaaa") + require.Contains(t, got, "tls=max-api-mysql-pin-"+fingerprint) reused, err := mysqlDSNWithPinnedCertificate(dsn) require.NoError(t, err) @@ -58,6 +58,17 @@ func TestMySQLDSNWithPinnedCertificateReplacesUnpinnedTLSOption(t *testing.T) { t.Setenv("MYSQL_TLS_CERT_SHA256", strings.Repeat("b", 64)) got, err := mysqlDSNWithPinnedCertificate("user:password@tcp(127.0.0.1:3306)/database?tls=skip-verify&parseTime=true") require.NoError(t, err) - require.Contains(t, got, "tls=max-api-mysql-pin-bbbbbbbbbbbbbbbb") + require.Contains(t, got, "tls=max-api-mysql-pin-"+strings.Repeat("b", 64)) require.NotContains(t, got, "tls=skip-verify") } + +func TestMySQLDSNWithPinnedCertificatePreservesDriverDSNParsing(t *testing.T) { + fingerprint := strings.Repeat("c", 64) + t.Setenv("MYSQL_TLS_CERT_SHA256", fingerprint) + dsn := "user:pa?ss@tcp(127.0.0.1:3306)/database?parseTime=true&loc=Asia%2FShanghai" + got, err := mysqlDSNWithPinnedCertificate(dsn) + require.NoError(t, err) + require.Contains(t, got, "tls=max-api-mysql-pin-"+fingerprint) + require.Contains(t, got, "parseTime=true") + require.Contains(t, got, "loc=Asia%2FShanghai") +} diff --git a/model/task_billing_external_test.go b/model/task_billing_external_test.go index 15035e4e..401b3f17 100644 --- a/model/task_billing_external_test.go +++ b/model/task_billing_external_test.go @@ -103,6 +103,7 @@ func openTaskBillingTestSQL(dialect, dsn string) (*sql.DB, string, error) { return nil, "", errTaskBillingUnsafeDatabase } config.TLS = &tls.Config{ + //nolint:gosec // VerifyConnection enforces the configured exact leaf SHA-256 pin. InsecureSkipVerify: true, MinVersion: tls.VersionTLS12, VerifyConnection: func(state tls.ConnectionState) error { diff --git a/pkg/taskplugin/manifest.go b/pkg/taskplugin/manifest.go index 1d716e27..36af44fa 100644 --- a/pkg/taskplugin/manifest.go +++ b/pkg/taskplugin/manifest.go @@ -123,10 +123,12 @@ func validateModels(models []string) (map[string]struct{}, error) { } set := make(map[string]struct{}, len(models)) for _, model := range models { - model = strings.TrimSpace(model) if model == "" { return nil, fmt.Errorf("task plugin model must not be empty") } + if model != strings.TrimSpace(model) { + return nil, fmt.Errorf("task plugin model %q must not have surrounding whitespace", model) + } if _, exists := set[model]; exists { return nil, fmt.Errorf("duplicate task plugin model %q", model) } @@ -194,6 +196,9 @@ func validateRoutes(routes []Route, models map[string]struct{}) error { if !identifierPattern.MatchString(route.Render) { return fmt.Errorf("query route %q requires a valid render hook", route.Path) } + if route.TaskIDParam != "" && !routeHasParameter(route.Path, route.TaskIDParam) { + return fmt.Errorf("query route %q task id parameter %q is not in the path", route.Path, route.TaskIDParam) + } } else { if !identifierPattern.MatchString(route.Decode) || !identifierPattern.MatchString(route.Render) { return fmt.Errorf("route %q requires valid decode and render hooks", route.Path) @@ -213,6 +218,15 @@ func validateRoutes(routes []Route, models map[string]struct{}) error { return nil } +func routeHasParameter(path, parameter string) bool { + for _, part := range strings.Split(path, "/") { + if strings.HasPrefix(part, ":") && strings.TrimPrefix(part, ":") == parameter { + return true + } + } + return false +} + func routeShape(rawPath string) (string, error) { if rawPath == "" || !strings.HasPrefix(rawPath, "/") || strings.ContainsAny(rawPath, "?#") { return "", fmt.Errorf("invalid task plugin route path %q", rawPath) @@ -318,8 +332,11 @@ func validateUsageSchema(schema map[string]UsageField) error { } func isASCII(value string) bool { + if !utf8.ValidString(value) { + return false + } for _, r := range value { - if r > 127 || !utf8.ValidString(value) { + if r > 127 { return false } } diff --git a/pkg/taskplugin/manifest_test.go b/pkg/taskplugin/manifest_test.go index 4a4daeb7..21bde791 100644 --- a/pkg/taskplugin/manifest_test.go +++ b/pkg/taskplugin/manifest_test.go @@ -97,3 +97,24 @@ func TestValidateManifestRejectsUndeclaredRouteModel(t *testing.T) { t.Fatal("expected undeclared route model to be rejected") } } + +func TestValidateManifestRejectsInvalidQueryTaskIDParameter(t *testing.T) { + manifest := validManifest() + manifest.Routes[1].TaskIDParam = "missing" + if err := ValidateManifest(manifest); err == nil { + t.Fatal("expected query route task id parameter to match a path parameter") + } + manifest = validManifest() + manifest.Routes[1].TaskIDParam = "task_id" + if err := ValidateManifest(manifest); err != nil { + t.Fatalf("expected matching query task id parameter to pass: %v", err) + } +} + +func TestValidateManifestRejectsPaddedModelNames(t *testing.T) { + manifest := validManifest() + manifest.Models[0] = " vendor-model" + if err := ValidateManifest(manifest); err == nil { + t.Fatal("expected padded model name to be rejected") + } +} diff --git a/pkg/taskplugin/registry_test.go b/pkg/taskplugin/registry_test.go index a968afe5..d3a7b4a8 100644 --- a/pkg/taskplugin/registry_test.go +++ b/pkg/taskplugin/registry_test.go @@ -95,7 +95,6 @@ func TestRegistryConcurrentPublicationPreservesAllUniquePlugins(t *testing.T) { var group sync.WaitGroup errors := make(chan error, count) for index := 0; index < count; index++ { - index := index group.Add(1) go func() { defer group.Done() diff --git a/tools/sqliteops/main.go b/tools/sqliteops/main.go index 2e80d623..72a28cd3 100644 --- a/tools/sqliteops/main.go +++ b/tools/sqliteops/main.go @@ -395,7 +395,7 @@ func writeManifest(root string, value manifest) error { return os.WriteFile(filepath.Join(root, manifestName), data, 0o600) } -func fingerprintFile(path string) (configurationHash, error) { +func fingerprintFile(path string) (result configurationHash, err error) { info, err := os.Stat(path) if err != nil { return configurationHash{}, err @@ -407,7 +407,11 @@ func fingerprintFile(path string) (configurationHash, error) { if err != nil { return configurationHash{}, err } - defer file.Close() + defer func() { + if closeErr := file.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() hash := sha256.New() if _, err := io.Copy(hash, file); err != nil { return configurationHash{}, err @@ -423,7 +427,12 @@ func copyRegularFile(source, destination string, mode os.FileMode) error { if err != nil { return err } - defer input.Close() + inputClosed := false + defer func() { + if !inputClosed { + _ = input.Close() + } + }() output, err := os.OpenFile(destination, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode) if err != nil { return err @@ -444,6 +453,10 @@ func copyRegularFile(source, destination string, mode os.FileMode) error { if err := output.Close(); err != nil { return err } + inputClosed = true + if err := input.Close(); err != nil { + return err + } remove = false return nil } @@ -518,5 +531,15 @@ func isWithin(root, path string) bool { func samePath(left, right string) bool { leftAbs, _ := filepath.Abs(left) rightAbs, _ := filepath.Abs(right) - return strings.EqualFold(filepath.Clean(leftAbs), filepath.Clean(rightAbs)) + leftClean := filepath.Clean(leftAbs) + rightClean := filepath.Clean(rightAbs) + if leftClean == rightClean { + return true + } + leftInfo, leftErr := os.Stat(leftClean) + rightInfo, rightErr := os.Stat(rightClean) + if leftErr != nil || rightErr != nil { + return false + } + return os.SameFile(leftInfo, rightInfo) } diff --git a/tools/sqliteops/main_test.go b/tools/sqliteops/main_test.go index 6c6a9480..00b3d269 100644 --- a/tools/sqliteops/main_test.go +++ b/tools/sqliteops/main_test.go @@ -105,6 +105,14 @@ func TestBackupRejectsDestinationInsideSource(t *testing.T) { require.ErrorContains(t, err, "source directory") } +func TestSamePathUsesExactOrExistingFileIdentity(t *testing.T) { + root := t.TempDir() + file := filepath.Join(root, "ledger.sqlite") + require.NoError(t, os.WriteFile(file, []byte("ledger"), 0o600)) + require.True(t, samePath(file, filepath.Join(root, ".", "ledger.sqlite"))) + require.False(t, samePath(file, filepath.Join(root, "other.sqlite"))) +} + func mustRead(t *testing.T, path string) []byte { t.Helper() data, err := os.ReadFile(path) From dc34eabbf319c669b97992661b3fd6a1a81b840f Mon Sep 17 00:00:00 2001 From: CSCITech Date: Sat, 17 Oct 2026 01:45:07 +0800 Subject: [PATCH 15/16] fix: close remaining PR62 review findings --- docs/installation/sqlite-operations.md | 4 +- model/mysql_migration_acceptance_test.go | 27 ++++++++++- model/sqlite_operations_acceptance_test.go | 19 ++++++++ model/task_billing_external_test.go | 2 +- pkg/taskplugin/registry.go | 10 +++- pkg/taskplugin/registry_test.go | 19 ++++++++ tools/sqliteops/main.go | 54 ++++++++++++++++++++-- tools/sqliteops/main_test.go | 45 ++++++++++++++++++ 8 files changed, 171 insertions(+), 9 deletions(-) diff --git a/docs/installation/sqlite-operations.md b/docs/installation/sqlite-operations.md index c14664ac..0db11e65 100644 --- a/docs/installation/sqlite-operations.md +++ b/docs/installation/sqlite-operations.md @@ -43,13 +43,13 @@ Verification rejects missing, modified, path-escaping, and unlisted files. Resto ## 业务快照验收 / Business snapshot rehearsal -仓库中的合成验收会使用当前模型创建主库和独立日志库，执行归档与两次隔离恢复，然后重新打开恢复文件，比较用户额度、Token、渠道用量、订阅周期与预消费、settlement、日志和 receipt。它只验证文件归档及当前 SQLite 模型数据的可读性，不代表真实生产库、跨库原子性或外部支付状态已经验证： +仓库中的合成验收会使用当前模型创建主库和独立日志库，执行归档与两次隔离恢复，然后重新打开恢复文件，比较用户额度、Token、渠道用量、订阅周期与预消费、settlement、日志和 receipt；主库还会在保持连接期间写入已提交但未检查点的真实 WAL 标记，并在恢复后核对。它只验证文件归档及当前 SQLite 模型数据的可读性，不代表真实生产库、跨库原子性或外部支付状态已经验证： ```powershell go test ./model -run TestSQLiteOperationsAcceptancePreservesBusinessSnapshot -count=1 ``` -The repository rehearsal creates a synthetic primary database and independent log database from the current models, archives them, restores them twice into isolated directories, reopens both copies, and compares wallet quota, token and channel usage, subscription/pre-consume state, settlement, logs, and receipts. It validates the archive and current SQLite model readability only; it does not establish production-database, cross-database atomicity, or external-payment correctness. +The repository rehearsal creates a synthetic primary database and independent log database from the current models, archives them, restores them twice into isolated directories, reopens both copies, and compares wallet quota, token and channel usage, subscription/pre-consume state, settlement, logs, and receipts. It also keeps the primary connection open while a committed, uncheckpointed WAL marker is archived and verifies that marker after restore. It validates the archive and current SQLite model readability only; it does not establish production-database, cross-database atomicity, or external-payment correctness. ## 边界 / Boundaries diff --git a/model/mysql_migration_acceptance_test.go b/model/mysql_migration_acceptance_test.go index 8b19f979..d4ddaf04 100644 --- a/model/mysql_migration_acceptance_test.go +++ b/model/mysql_migration_acceptance_test.go @@ -43,6 +43,10 @@ import ( var mysqlMigrationAcceptanceDatabase = regexp.MustCompile(`^maxapi_mysql_acceptance_[a-z0-9_]{1,32}$`) +func mysqlAcceptanceDatabaseAllowed(name, override string, pattern *regexp.Regexp) bool { + return pattern.MatchString(name) || (override != "" && name == override) +} + func TestMySQLSchemaCompatibilityReadOnly(t *testing.T) { dsn := strings.TrimSpace(os.Getenv("TEST_MYSQL_SCHEMA_AUDIT_DSN")) if dsn == "" { @@ -51,7 +55,7 @@ func TestMySQLSchemaCompatibilityReadOnly(t *testing.T) { config, err := mysqldriver.ParseDSN(dsn) require.NoError(t, err) allowedDatabase := strings.TrimSpace(os.Getenv("TEST_MYSQL_SCHEMA_AUDIT_DATABASE")) - if !mysqlMigrationAcceptanceDatabase.MatchString(config.DBName) && config.DBName != allowedDatabase { + if !mysqlAcceptanceDatabaseAllowed(config.DBName, allowedDatabase, mysqlMigrationAcceptanceDatabase) { t.Fatalf("refusing MySQL schema audit target with unsafe database name") } @@ -83,7 +87,7 @@ func TestMySQLStartupMigrationOnEmptyDatabase(t *testing.T) { config, err := mysqldriver.ParseDSN(dsn) require.NoError(t, err) allowedDatabase := strings.TrimSpace(os.Getenv("TEST_MYSQL_MIGRATION_DATABASE")) - if !mysqlMigrationAcceptanceDatabase.MatchString(config.DBName) && config.DBName != allowedDatabase { + if !mysqlAcceptanceDatabaseAllowed(config.DBName, allowedDatabase, mysqlMigrationAcceptanceDatabase) { t.Fatalf("refusing MySQL migration target with unsafe database name") } if strings.TrimSpace(os.Getenv("TEST_MYSQL_TLS_CERT_SHA256")) == "" && config.Net != "unix" && (config.TLS == nil || config.TLS.InsecureSkipVerify) { @@ -130,6 +134,25 @@ func TestMySQLStartupMigrationOnEmptyDatabase(t *testing.T) { require.True(t, db.Migrator().HasIndex(&BillingSettlement{}, "idx_billing_settlement_reconciliation")) } +func TestMySQLAcceptanceDatabaseGuardRequiresExplicitOverride(t *testing.T) { + tests := []struct { + name string + pattern *regexp.Regexp + valid string + }{ + {name: "task billing", pattern: taskBillingTestDatabaseName, valid: "maxapi_task_billing_test_fixture"}, + {name: "migration", pattern: mysqlMigrationAcceptanceDatabase, valid: "maxapi_mysql_acceptance_fixture"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + require.False(t, mysqlAcceptanceDatabaseAllowed("", "", test.pattern)) + require.False(t, mysqlAcceptanceDatabaseAllowed("production", "", test.pattern)) + require.True(t, mysqlAcceptanceDatabaseAllowed("dedicated_override", "dedicated_override", test.pattern)) + require.True(t, mysqlAcceptanceDatabaseAllowed(test.valid, "", test.pattern)) + }) + } +} + func testEnv(name string) string { return strings.TrimSpace(os.Getenv(name)) } diff --git a/model/sqlite_operations_acceptance_test.go b/model/sqlite_operations_acceptance_test.go index 5f727f4c..b146f7a9 100644 --- a/model/sqlite_operations_acceptance_test.go +++ b/model/sqlite_operations_acceptance_test.go @@ -119,6 +119,8 @@ func TestSQLiteOperationsAcceptancePreservesBusinessSnapshot(t *testing.T) { seedSQLiteOperationsDataset(t, primaryPath, logPath) before := readSQLiteOperationsSnapshot(t, primaryPath, logPath) + walDB := seedSQLiteOperationsWALFixture(t, primaryPath) + t.Cleanup(func() { require.NoError(t, closeSQLiteOperationsDatabase(walDB)) }) archivePath := filepath.Join(workDir, "archive") runSQLiteOperationsCLI(t, root, "backup", "--quiesced", "--primary", primaryPath, "--log", logPath, "--config", configPath, "--destination", archivePath) @@ -133,6 +135,11 @@ func TestSQLiteOperationsAcceptancePreservesBusinessSnapshot(t *testing.T) { filepath.Join(firstRestore, "primary", filepath.Base(primaryPath)), filepath.Join(firstRestore, "log", filepath.Base(logPath)), ) + restoredDB := openSQLiteOperationsDatabase(t, filepath.Join(firstRestore, "primary", filepath.Base(primaryPath))) + var walMarker Option + require.NoError(t, restoredDB.Where("key = ?", "sqliteops-wal-marker").First(&walMarker).Error) + require.Equal(t, "committed-in-wal", walMarker.Value) + require.NoError(t, closeSQLiteOperationsDatabase(restoredDB)) // Reopening the restored files models a second cold start and catches // archives that are readable only while the first connection is open. secondStartSnapshot := readSQLiteOperationsSnapshot(t, @@ -167,6 +174,7 @@ func seedSQLiteOperationsDataset(t *testing.T, primaryPath, logPath string) { require.NoError(t, db.AutoMigrate( &User{}, &Token{}, &Channel{}, &SubscriptionPlan{}, &UserSubscription{}, &SubscriptionPreConsumeRecord{}, &BillingPreConsumeSelection{}, &BillingSettlement{}, + &Option{}, )) require.NoError(t, db.Create(&User{ Id: userID, Username: "sqliteops-acceptance", Password: "synthetic-password", @@ -226,6 +234,17 @@ func seedSQLiteOperationsDataset(t *testing.T, primaryPath, logPath string) { require.NoError(t, closeSQLiteOperationsDatabase(logDB)) } +func seedSQLiteOperationsWALFixture(t *testing.T, primaryPath string) *gorm.DB { + db := openSQLiteOperationsDatabase(t, primaryPath) + require.NoError(t, db.Exec("PRAGMA journal_mode=WAL").Error) + require.NoError(t, db.Exec("PRAGMA wal_autocheckpoint=0").Error) + require.NoError(t, db.Create(&Option{Key: "sqliteops-wal-marker", Value: "committed-in-wal"}).Error) + walInfo, err := os.Stat(primaryPath + "-wal") + require.NoError(t, err, "the committed fixture row must remain in a real WAL sidecar") + require.Greater(t, walInfo.Size(), int64(0), "the committed fixture row must remain in a non-empty WAL sidecar") + return db +} + func readSQLiteOperationsSnapshot(t *testing.T, primaryPath, logPath string) sqliteOperationsBusinessSnapshot { const ( userID = 91001 diff --git a/model/task_billing_external_test.go b/model/task_billing_external_test.go index 401b3f17..7cb5f9a8 100644 --- a/model/task_billing_external_test.go +++ b/model/task_billing_external_test.go @@ -86,7 +86,7 @@ func openTaskBillingTestSQL(dialect, dsn string) (*sql.DB, string, error) { return nil, "", err } allowedDatabase := strings.TrimSpace(os.Getenv("TEST_TASK_BILLING_MYSQL_DATABASE")) - if !taskBillingTestDatabaseName.MatchString(config.DBName) && config.DBName != allowedDatabase { + if !mysqlAcceptanceDatabaseAllowed(config.DBName, allowedDatabase, taskBillingTestDatabaseName) { return nil, "", errTaskBillingUnsafeDatabase } // External contract tests may never put database credentials on a diff --git a/pkg/taskplugin/registry.go b/pkg/taskplugin/registry.go index 55923d51..0dc42aa9 100644 --- a/pkg/taskplugin/registry.go +++ b/pkg/taskplugin/registry.go @@ -93,7 +93,15 @@ func (registry *Registry) Publish(manifest Manifest) (uint64, error) { } for method := range next.byMethod { sort.SliceStable(next.byMethod[method], func(i, j int) bool { - return routeSpecificity(next.byMethod[method][i].route.Path) > routeSpecificity(next.byMethod[method][j].route.Path) + a, b := next.byMethod[method][i], next.byMethod[method][j] + sa, sb := routeSpecificity(a.route.Path), routeSpecificity(b.route.Path) + if sa != sb { + return sa > sb + } + if a.pluginKey != b.pluginKey { + return a.pluginKey < b.pluginKey + } + return a.route.Path < b.route.Path }) } if registry.state.CompareAndSwap(current, next) { diff --git a/pkg/taskplugin/registry_test.go b/pkg/taskplugin/registry_test.go index d3a7b4a8..7944b32c 100644 --- a/pkg/taskplugin/registry_test.go +++ b/pkg/taskplugin/registry_test.go @@ -62,6 +62,25 @@ func TestRegistryPublishesAtomicSnapshotAndMatchesRoute(t *testing.T) { } } +func TestRegistryBreaksEqualSpecificityTiesDeterministically(t *testing.T) { + for _, order := range [][]string{{"beta", "alpha"}, {"alpha", "beta"}} { + registry := NewRegistry() + routes := map[string]string{ + "alpha": "/jobs/:id", + "beta": "/:kind/latest", + } + for _, key := range order { + if _, err := registry.Publish(registryManifest(key, routes[key])); err != nil { + t.Fatalf("publish %s: %v", key, err) + } + } + match, ok := registry.Snapshot().MatchRoute("GET", "/jobs/latest", "vendor-model") + if !ok || match.PluginKey != "alpha" { + t.Fatalf("expected alpha to win equal-specificity tie, got %#v, %v", match, ok) + } + } +} + func TestRegistryRejectsDuplicatePluginAndRouteConflict(t *testing.T) { registry := NewRegistry() if _, err := registry.Publish(registryManifest("alpha", "/jobs/:task_id")); err != nil { diff --git a/tools/sqliteops/main.go b/tools/sqliteops/main.go index 72a28cd3..20debb4c 100644 --- a/tools/sqliteops/main.go +++ b/tools/sqliteops/main.go @@ -338,6 +338,9 @@ func verifyArchive(root string) error { if err != nil { return err } + if err := requireRegularFile(path); err != nil { + return fmt.Errorf("verify %s: %w", item.ArchivePath, err) + } actual, err := fingerprintFile(path) if err != nil { return fmt.Errorf("verify %s: %w", item.ArchivePath, err) @@ -383,6 +386,22 @@ func readManifest(root string) (manifest, error) { if result.SchemaVersion != manifestVersion || len(result.Files) == 0 { return manifest{}, errors.New("unsupported or empty archive manifest") } + roles := map[string]bool{} + mainRoles := map[string]bool{} + for _, item := range result.Files { + for _, role := range item.Roles { + roles[role] = true + if !strings.HasSuffix(item.ArchivePath, "-wal") && !strings.HasSuffix(item.ArchivePath, "-shm") { + mainRoles[role] = true + } + } + } + if !mainRoles["primary"] { + return manifest{}, errors.New("archive manifest is missing the primary database") + } + if roles["log"] && !mainRoles["log"] { + return manifest{}, errors.New("archive manifest is missing the log database") + } return result, nil } @@ -513,9 +532,38 @@ func safeArchivePath(root, relative string) (string, error) { } func pathsOverlap(left, right string) bool { - leftAbs, _ := filepath.Abs(left) - rightAbs, _ := filepath.Abs(right) - return isWithin(leftAbs, rightAbs) || isWithin(rightAbs, leftAbs) + leftResolved := resolveOverlapPath(left) + rightResolved := resolveOverlapPath(right) + return isWithin(leftResolved, rightResolved) || isWithin(rightResolved, leftResolved) +} + +func resolveOverlapPath(path string) string { + abs, err := filepath.Abs(path) + if err != nil { + return filepath.Clean(path) + } + current := filepath.Clean(abs) + var missing []string + for { + if _, err := os.Lstat(current); err == nil { + resolved, err := filepath.EvalSymlinks(current) + if err != nil { + return filepath.Clean(abs) + } + for index := len(missing) - 1; index >= 0; index-- { + resolved = filepath.Join(resolved, missing[index]) + } + return filepath.Clean(resolved) + } else if !os.IsNotExist(err) { + return filepath.Clean(abs) + } + parent := filepath.Dir(current) + if parent == current { + return filepath.Clean(abs) + } + missing = append(missing, filepath.Base(current)) + current = parent + } } func isWithin(root, path string) bool { diff --git a/tools/sqliteops/main_test.go b/tools/sqliteops/main_test.go index 00b3d269..573bc8a1 100644 --- a/tools/sqliteops/main_test.go +++ b/tools/sqliteops/main_test.go @@ -93,6 +93,35 @@ func TestVerifyRejectsTamperingAndUnlistedFiles(t *testing.T) { require.ErrorContains(t, verifyArchive(archive2), "unlisted file") } +func TestVerifyRejectsManifestWithoutPrimaryDatabase(t *testing.T) { + primary := filepath.Join(t.TempDir(), "ledger.sqlite") + require.NoError(t, os.WriteFile(primary, []byte("ledger"), 0o600)) + archive := filepath.Join(t.TempDir(), "archive") + require.NoError(t, runBackup([]string{"--quiesced", "--primary", primary, "--destination", archive})) + value, err := readManifest(archive) + require.NoError(t, err) + for index := range value.Files { + value.Files[index].Roles = []string{"log"} + } + require.NoError(t, writeManifest(archive, value)) + require.ErrorContains(t, verifyArchive(archive), "missing the primary database") +} + +func TestVerifyRejectsSymlinkedArchiveFile(t *testing.T) { + primary := filepath.Join(t.TempDir(), "ledger.sqlite") + require.NoError(t, os.WriteFile(primary, []byte("ledger"), 0o600)) + archive := filepath.Join(t.TempDir(), "archive") + require.NoError(t, runBackup([]string{"--quiesced", "--primary", primary, "--destination", archive})) + archiveFile := filepath.Join(archive, "primary", "ledger.sqlite") + target := filepath.Join(t.TempDir(), "target.sqlite") + require.NoError(t, os.WriteFile(target, []byte("ledger"), 0o600)) + require.NoError(t, os.Remove(archiveFile)) + if err := os.Symlink(target, archiveFile); err != nil { + t.Skipf("symlink creation is unavailable: %v", err) + } + require.ErrorContains(t, verifyArchive(archive), "regular and not a symlink") +} + func TestBackupRejectsDestinationInsideSource(t *testing.T) { source := t.TempDir() primary := filepath.Join(source, "ledger.sqlite") @@ -105,6 +134,22 @@ func TestBackupRejectsDestinationInsideSource(t *testing.T) { require.ErrorContains(t, err, "source directory") } +func TestBackupRejectsDestinationInsideSymlinkedSource(t *testing.T) { + source := t.TempDir() + primary := filepath.Join(source, "ledger.sqlite") + require.NoError(t, os.WriteFile(primary, []byte("ledger"), 0o600)) + alias := filepath.Join(t.TempDir(), "source-link") + if err := os.Symlink(source, alias); err != nil { + t.Skipf("symlink creation is unavailable: %v", err) + } + err := runBackup([]string{ + "--quiesced", + "--primary", primary, + "--destination", filepath.Join(alias, "archive"), + }) + require.ErrorContains(t, err, "source directory") +} + func TestSamePathUsesExactOrExistingFileIdentity(t *testing.T) { root := t.TempDir() file := filepath.Join(root, "ledger.sqlite") From f00fddee8dde7fb7d5c61845b0e66f6b382662da Mon Sep 17 00:00:00 2001 From: CSCITech Date: Sat, 17 Oct 2026 02:37:54 +0800 Subject: [PATCH 16/16] fix: make sqlite operations publication atomic --- model/sqlite_upgrade_test.go | 2 +- tools/sqliteops/main.go | 72 +++++++++++++++++++++++++++++++++--- tools/sqliteops/main_test.go | 19 ++++++++++ 3 files changed, 86 insertions(+), 7 deletions(-) diff --git a/model/sqlite_upgrade_test.go b/model/sqlite_upgrade_test.go index e58fba8e..0e1e052c 100644 --- a/model/sqlite_upgrade_test.go +++ b/model/sqlite_upgrade_test.go @@ -261,8 +261,8 @@ func TestSQLiteUpgradeProcessHelper(t *testing.T) { common.LogConsumeEnabled, common.DebugEnabled = true, false logRetryMarkerBackfillAsyncRunner = func(fn func()) { fn() } require.NoError(t, InitDB()) - t.Cleanup(func() { require.NoError(t, CloseDB()) }) require.NoError(t, InitLogDB()) + t.Cleanup(func() { require.NoError(t, CloseDB()) }) DB.Logger, LOG_DB.Logger = logger.Default.LogMode(logger.Silent), logger.Default.LogMode(logger.Silent) var history User require.NoError(t, DB.First(&history, 8700).Error) diff --git a/tools/sqliteops/main.go b/tools/sqliteops/main.go index 20debb4c..1d83820b 100644 --- a/tools/sqliteops/main.go +++ b/tools/sqliteops/main.go @@ -127,6 +127,16 @@ func runBackup(args []string) error { if err != nil { return err } + stageRoot, err := createStagingDirectory(root) + if err != nil { + return err + } + published := false + defer func() { + if !published { + _ = os.RemoveAll(stageRoot) + } + }() sources, err := collectSources(*primary, *logPath) if err != nil { return err @@ -137,7 +147,7 @@ func runBackup(args []string) error { Tool: "MAX-API sqliteops", } for _, source := range sources { - entry, err := archiveSource(root, source) + entry, err := archiveSource(stageRoot, source) if err != nil { return err } @@ -152,10 +162,17 @@ func runBackup(args []string) error { m.Configuration = &fingerprint } sort.Slice(m.Files, func(i, j int) bool { return m.Files[i].ArchivePath < m.Files[j].ArchivePath }) - if err := writeManifest(root, m); err != nil { + if err := writeManifest(stageRoot, m); err != nil { return err } - return verifyArchive(root) + if err := verifyArchive(stageRoot); err != nil { + return err + } + if err := publishStagingDirectory(stageRoot, root); err != nil { + return err + } + published = true + return nil } func runVerify(args []string) error { @@ -203,9 +220,16 @@ func runRestore(args []string) error { if pathsOverlap(backupRoot, targetRoot) { return errors.New("restore target must not overlap the backup archive") } - if err := os.MkdirAll(targetRoot, 0o700); err != nil { + stageRoot, err := createStagingDirectory(targetRoot) + if err != nil { return err } + published := false + defer func() { + if !published { + _ = os.RemoveAll(stageRoot) + } + }() m, err := readManifest(backupRoot) if err != nil { return err @@ -215,7 +239,7 @@ func runRestore(args []string) error { if err != nil { return err } - destination, err := safeArchivePath(targetRoot, item.ArchivePath) + destination, err := safeArchivePath(stageRoot, item.ArchivePath) if err != nil { return err } @@ -226,7 +250,14 @@ func runRestore(args []string) error { return err } } - return copyRegularFile(filepath.Join(backupRoot, manifestName), filepath.Join(targetRoot, manifestName), 0o600) + if err := copyRegularFile(filepath.Join(backupRoot, manifestName), filepath.Join(stageRoot, manifestName), 0o600); err != nil { + return err + } + if err := publishStagingDirectory(stageRoot, targetRoot); err != nil { + return err + } + published = true + return nil } func collectSources(primary, logPath string) ([]sourceFile, error) { @@ -320,6 +351,35 @@ func prepareDestination(destination, primary, logPath, config string) (string, e return root, nil } +func createStagingDirectory(destination string) (string, error) { + parent := filepath.Dir(destination) + if err := os.MkdirAll(parent, 0o700); err != nil { + return "", err + } + return os.MkdirTemp(parent, "."+filepath.Base(destination)+".tmp-") +} + +func publishStagingDirectory(stage, destination string) error { + if info, err := os.Stat(destination); err == nil { + if !info.IsDir() { + return fmt.Errorf("destination is not a directory: %s", destination) + } + entries, err := os.ReadDir(destination) + if err != nil { + return err + } + if len(entries) > 0 { + return fmt.Errorf("destination must remain empty until publish: %s", destination) + } + if err := os.Remove(destination); err != nil { + return err + } + } else if !os.IsNotExist(err) { + return err + } + return os.Rename(stage, destination) +} + func verifyArchive(root string) error { m, err := readManifest(root) if err != nil { diff --git a/tools/sqliteops/main_test.go b/tools/sqliteops/main_test.go index 573bc8a1..8da16293 100644 --- a/tools/sqliteops/main_test.go +++ b/tools/sqliteops/main_test.go @@ -134,6 +134,25 @@ func TestBackupRejectsDestinationInsideSource(t *testing.T) { require.ErrorContains(t, err, "source directory") } +func TestBackupFailureLeavesDestinationEmpty(t *testing.T) { + primary := filepath.Join(t.TempDir(), "ledger.sqlite") + config := filepath.Join(t.TempDir(), "config-directory") + require.NoError(t, os.WriteFile(primary, []byte("ledger"), 0o600)) + require.NoError(t, os.Mkdir(config, 0o700)) + destination := filepath.Join(t.TempDir(), "archive") + require.NoError(t, os.Mkdir(destination, 0o700)) + err := runBackup([]string{ + "--quiesced", + "--primary", primary, + "--config", config, + "--destination", destination, + }) + require.ErrorContains(t, err, "not a regular file") + entries, readErr := os.ReadDir(destination) + require.NoError(t, readErr) + require.Empty(t, entries) +} + func TestBackupRejectsDestinationInsideSymlinkedSource(t *testing.T) { source := t.TempDir() primary := filepath.Join(source, "ledger.sqlite")