From cb2f4f5f3fec250896472807e03e8204806fdd8d Mon Sep 17 00:00:00 2001 From: Robin Gagnon Date: Thu, 15 Oct 2026 10:04:52 -0500 Subject: [PATCH 1/3] fix: handle turso 429 load-shed during generate the generate job failed in production when turso returned 429 load-shed on a colorscheme_groups insert. the write had no retry and the panic aborted the whole batch. treat 429 rate-limit errors as transient so writes retry, run the generate write in the retrying transaction, and record a per-repo error instead of failing the job. --- cli/generate.go | 14 ++- internal/database/repository.go | 157 ++++++++++++++------------------ internal/database/retry.go | 6 ++ internal/database/retry_test.go | 10 ++ 4 files changed, 97 insertions(+), 90 deletions(-) diff --git a/cli/generate.go b/cli/generate.go index 17eb155..bc0c8c0 100644 --- a/cli/generate.go +++ b/cli/generate.go @@ -98,7 +98,15 @@ func Generate(force bool, debug bool, repoKey string) map[string]interface{} { } repository.Colorschemes = buildRepositoryColorschemes(data, repository.Name) - updateRepositoryAfterGenerate(repository) + if err := updateRepositoryAfterGenerate(repository); err != nil { + log.Printf("Error writing generate data: %s", err) + repositoryErrorCount++ + repositoryErrorSamples = appendRepositoryErrorSample(repositoryErrorSamples, repository, err) + if eventErr := database.CreateRepositoryGenerateErrorEvent(repository.ID, err.Error()); eventErr != nil { + log.Printf("Error creating generate failure event: %s", eventErr) + } + continue + } } cleanUp() @@ -110,10 +118,10 @@ func Generate(force bool, debug bool, repoKey string) map[string]interface{} { } } -func updateRepositoryAfterGenerate(repository repoHelper.Repository) { +func updateRepositoryAfterGenerate(repository repoHelper.Repository) error { log.Printf("Generated %d colorschemes", len(repository.Colorschemes)) data := getGenerateData(repository) - database.UpdateRepositoryFromGenerate(repository.ID, data) + return database.UpdateRepositoryFromGenerate(repository.ID, data) } // Initializes a temporary directory for runtime configuration files diff --git a/internal/database/repository.go b/internal/database/repository.go index cf3f1b7..169f2eb 100644 --- a/internal/database/repository.go +++ b/internal/database/repository.go @@ -5,6 +5,7 @@ import ( "database/sql" "encoding/json" "errors" + "fmt" "log" "strings" "time" @@ -411,101 +412,83 @@ func placeholders(count int) string { } // UpdateRepositoryFromGenerate updates a repository with generate job data. -func UpdateRepositoryFromGenerate(id int64, data GenerateData) { - tx, err := beginWithTransientRetry() - if err != nil { - panic(err) - } - defer func() { - _ = tx.Rollback() - }() - - _, err = tx.Exec("DELETE FROM colorschemes WHERE repository_id = ?", id) - if err != nil { - log.Printf("Error deleting colorschemes: %s", err) - panic(err) - } - - hasDark, hasLight := 0, 0 - for _, scheme := range data.Colorschemes { - if len(scheme.Data.Dark) > 0 { - hasDark = 1 - } - if len(scheme.Data.Light) > 0 { - hasLight = 1 +// The whole transaction retries on transient errors (including Turso 429 +// load-shed). Returns an error so callers can isolate a single repository +// failure instead of aborting the job. +func UpdateRepositoryFromGenerate(id int64, data GenerateData) error { + return runRepositoryWriteTransaction(func(ctx context.Context, tx *sql.Tx) error { + if _, err := tx.ExecContext(ctx, "DELETE FROM colorschemes WHERE repository_id = ?", id); err != nil { + return fmt.Errorf("delete colorschemes: %w", err) } - } - _, err = tx.Exec("UPDATE repositories SET has_dark = ?, has_light = ? WHERE id = ?", hasDark, hasLight, id) - if err != nil { - log.Printf("Error updating has_dark/has_light: %s", err) - panic(err) - } - for _, scheme := range data.Colorschemes { - result, err := tx.Exec("INSERT INTO colorschemes (repository_id, name) VALUES (?, ?)", id, scheme.Name) - if err != nil { - log.Printf("Error inserting colorscheme: %s", err) - panic(err) + hasDark, hasLight := 0, 0 + for _, scheme := range data.Colorschemes { + if len(scheme.Data.Dark) > 0 { + hasDark = 1 + } + if len(scheme.Data.Light) > 0 { + hasLight = 1 + } } - schemeID, err := result.LastInsertId() - if err != nil { - panic(err) + if _, err := tx.ExecContext(ctx, "UPDATE repositories SET has_dark = ?, has_light = ? WHERE id = ?", hasDark, hasLight, id); err != nil { + return fmt.Errorf("update has_dark/has_light: %w", err) } - for _, bg := range []struct { - value repository.BackgroundValue - groups []repository.ColorschemeGroup - }{ - {repository.LightBackground, scheme.Data.Light}, - {repository.DarkBackground, scheme.Data.Dark}, - } { - for _, group := range bg.groups { - _, err = tx.Exec(` - INSERT INTO colorscheme_groups ( - colorscheme_id, - background, - name, - hex_code, - bold, - italic, - underline, - undercurl, - underdouble, - underdotted, - underdashed, - strikethrough, - reverse - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - schemeID, - bg.value, - group.Name, - group.HexCode, - group.Bold, - group.Italic, - group.Underline, - group.Undercurl, - group.Underdouble, - group.Underdotted, - group.Underdashed, - group.Strikethrough, - group.Reverse, - ) - if err != nil { - log.Printf("Error inserting colorscheme group: %s", err) - panic(err) + + for _, scheme := range data.Colorschemes { + result, err := tx.ExecContext(ctx, "INSERT INTO colorschemes (repository_id, name) VALUES (?, ?)", id, scheme.Name) + if err != nil { + return fmt.Errorf("insert colorscheme: %w", err) + } + schemeID, err := result.LastInsertId() + if err != nil { + return fmt.Errorf("colorscheme last insert id: %w", err) + } + for _, bg := range []struct { + value repository.BackgroundValue + groups []repository.ColorschemeGroup + }{ + {repository.LightBackground, scheme.Data.Light}, + {repository.DarkBackground, scheme.Data.Dark}, + } { + for _, group := range bg.groups { + if _, err := tx.ExecContext(ctx, ` + INSERT INTO colorscheme_groups ( + colorscheme_id, + background, + name, + hex_code, + bold, + italic, + underline, + undercurl, + underdouble, + underdotted, + underdashed, + strikethrough, + reverse + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + schemeID, + bg.value, + group.Name, + group.HexCode, + group.Bold, + group.Italic, + group.Underline, + group.Undercurl, + group.Underdouble, + group.Underdotted, + group.Underdashed, + group.Strikethrough, + group.Reverse, + ); err != nil { + return fmt.Errorf("insert colorscheme group: %w", err) + } } } } - } - err = createRepositoryJobEvent(tx, id, jobGenerate, jobStatusSuccess, "", time.Now().UTC()) - if err != nil { - log.Printf("Error creating repository job event: %s", err) - panic(err) - } - - if err := tx.Commit(); err != nil { - panic(err) - } + return createRepositoryJobEventsContext(ctx, tx, []int64{id}, jobGenerate, jobStatusSuccess, "", time.Now().UTC()) + }) } type repositoryJobEventExecutor interface { diff --git a/internal/database/retry.go b/internal/database/retry.go index a7d3eb6..7f6f15e 100644 --- a/internal/database/retry.go +++ b/internal/database/retry.go @@ -30,6 +30,12 @@ func isTransientLibSQLError(err error) bool { "sqlite_busy", "database is locked", "context deadline exceeded", + "too many requests", + "429", + "load-shed", + "load shed", + "rate limit", + "throttle", } { if strings.Contains(errMsg, marker) { return true diff --git a/internal/database/retry_test.go b/internal/database/retry_test.go index 0dc4ba5..3e8aba3 100644 --- a/internal/database/retry_test.go +++ b/internal/database/retry_test.go @@ -37,6 +37,16 @@ func TestIsTransientLibSQLError(t *testing.T) { err: testError("error code = 2: Error executing statement: Hrana: `unexpected response: `Unexpected multiple responses from server"), want: true, }, + { + name: "turso load-shed 429 is transient", + err: testError("error code = 3: Error preparing statement: Hrana: `api error: `status=429 Too Many Requests, body={\"error\":\"load-shed for host vimcolorschemes-reobin.aws-us-east-1.turso.io\"}``"), + want: true, + }, + { + name: "rate limit is transient", + err: testError("rate limit exceeded, retry later"), + want: true, + }, { name: "constraint failure is not transient", err: testError("UNIQUE constraint failed"), From cfbf6c6b95f8869c1d418ca81e0804b8687edcc3 Mon Sep 17 00:00:00 2001 From: Robin Gagnon Date: Thu, 15 Oct 2026 10:30:45 -0500 Subject: [PATCH 2/3] Fix CI lint errors from Turso 429 change --- internal/database/repository_test.go | 38 ++++++++++++++++++++-------- internal/database/retry.go | 27 -------------------- 2 files changed, 27 insertions(+), 38 deletions(-) diff --git a/internal/database/repository_test.go b/internal/database/repository_test.go index b7886bc..e5cfa37 100644 --- a/internal/database/repository_test.go +++ b/internal/database/repository_test.go @@ -792,7 +792,7 @@ func TestUpdateRepositoryFromGenerate(t *testing.T) { setupTestDB(t) insertTestRepo(t, 1, "owner", "repo") - UpdateRepositoryFromGenerate(1, GenerateData{ + if err := UpdateRepositoryFromGenerate(1, GenerateData{ Colorschemes: []repository.Colorscheme{ { Name: "myscheme", @@ -817,7 +817,9 @@ func TestUpdateRepositoryFromGenerate(t *testing.T) { }, }, }, - }) + }); err != nil { + t.Fatalf("UpdateRepositoryFromGenerate: %v", err) + } repo, err := GetRepository("owner/repo") if err != nil { @@ -850,12 +852,16 @@ func TestUpdateRepositoryFromGenerate(t *testing.T) { setupTestDB(t) insertTestRepo(t, 1, "owner", "repo") - UpdateRepositoryFromGenerate(1, GenerateData{ + if err := UpdateRepositoryFromGenerate(1, GenerateData{ Colorschemes: []repository.Colorscheme{{Name: "scheme1"}}, - }) - UpdateRepositoryFromGenerate(1, GenerateData{ + }); err != nil { + t.Fatalf("UpdateRepositoryFromGenerate: %v", err) + } + if err := UpdateRepositoryFromGenerate(1, GenerateData{ Colorschemes: []repository.Colorscheme{{Name: "scheme2"}}, - }) + }); err != nil { + t.Fatalf("UpdateRepositoryFromGenerate: %v", err) + } repo, err := GetRepository("owner/repo") if err != nil { @@ -873,7 +879,9 @@ func TestUpdateRepositoryFromGenerate(t *testing.T) { setupTestDB(t) insertTestRepo(t, 1, "owner", "repo") - UpdateRepositoryFromGenerate(1, GenerateData{}) + if err := UpdateRepositoryFromGenerate(1, GenerateData{}); err != nil { + t.Fatalf("UpdateRepositoryFromGenerate: %v", err) + } var eventCount int err := db.QueryRow(`SELECT COUNT(*) FROM repository_job_events WHERE repository_id = 1 AND job = 'generate' AND status = 'success'`).Scan(&eventCount) @@ -889,8 +897,12 @@ func TestUpdateRepositoryFromGenerate(t *testing.T) { setupTestDB(t) insertTestRepo(t, 1, "owner", "repo") - UpdateRepositoryFromGenerate(1, GenerateData{}) - UpdateRepositoryFromGenerate(1, GenerateData{}) + if err := UpdateRepositoryFromGenerate(1, GenerateData{}); err != nil { + t.Fatalf("UpdateRepositoryFromGenerate: %v", err) + } + if err := UpdateRepositoryFromGenerate(1, GenerateData{}); err != nil { + t.Fatalf("UpdateRepositoryFromGenerate: %v", err) + } var eventCount int err := db.QueryRow(`SELECT COUNT(*) FROM repository_job_events WHERE repository_id = 1 AND job = 'generate' AND status = 'success'`).Scan(&eventCount) @@ -935,8 +947,12 @@ func TestUpdateRepositoryFromGenerate(t *testing.T) { if err := CreateRepositoryGenerateErrorEvent(1, "second"); err != nil { t.Fatalf("CreateRepositoryGenerateErrorEvent: %v", err) } - UpdateRepositoryFromGenerate(1, GenerateData{}) - UpdateRepositoryFromGenerate(1, GenerateData{}) + if err := UpdateRepositoryFromGenerate(1, GenerateData{}); err != nil { + t.Fatalf("UpdateRepositoryFromGenerate: %v", err) + } + if err := UpdateRepositoryFromGenerate(1, GenerateData{}); err != nil { + t.Fatalf("UpdateRepositoryFromGenerate: %v", err) + } var errorCount int err := db.QueryRow(`SELECT COUNT(*) FROM repository_job_events WHERE repository_id = 1 AND job = 'generate' AND status = 'error'`).Scan(&errorCount) diff --git a/internal/database/retry.go b/internal/database/retry.go index 7f6f15e..e713f66 100644 --- a/internal/database/retry.go +++ b/internal/database/retry.go @@ -128,33 +128,6 @@ func queryWithTransientRetry(query string, args ...any) (*sql.Rows, error) { return nil, lastErr } -func beginWithTransientRetry() (*sql.Tx, error) { - var lastErr error - for attempt := 0; attempt <= transientLibSQLRetryAttempts; attempt++ { - tx, err := db.Begin() - if err == nil { - return tx, nil - } - - if !isTransientLibSQLError(err) { - return nil, err - } - - lastErr = err - if attempt == transientLibSQLRetryAttempts { - break - } - - log.Printf("DB begin failed with transient libsql error, retrying: %s", err) - if pingErr := pingWithTimeout(); pingErr != nil { - return nil, fmt.Errorf("ping before begin retry: %w", pingErr) - } - time.Sleep(transientLibSQLRetryBackoff(attempt)) - } - - return nil, lastErr -} - func pingWithTimeout() error { ctx, cancel := context.WithTimeout(context.Background(), dbOperationTimeout) defer cancel() From 73574b0759db460e337b8e31b89ff6de4779bb28 Mon Sep 17 00:00:00 2001 From: Robin Gagnon Date: Thu, 15 Oct 2026 10:45:57 -0500 Subject: [PATCH 3/3] Tighten 429 matching and cover generate write failures --- internal/database/repository_test.go | 40 ++++++++++++++++++++++++++++ internal/database/retry.go | 3 ++- internal/database/retry_test.go | 5 ++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/internal/database/repository_test.go b/internal/database/repository_test.go index e5cfa37..2988f7b 100644 --- a/internal/database/repository_test.go +++ b/internal/database/repository_test.go @@ -875,6 +875,46 @@ func TestUpdateRepositoryFromGenerate(t *testing.T) { } }) + t.Run("returns an error and leaves stored colorschemes untouched on failure", func(t *testing.T) { + setupTestDB(t) + insertTestRepo(t, 1, "owner", "repo") + + if err := UpdateRepositoryFromGenerate(1, GenerateData{ + Colorschemes: []repository.Colorscheme{{Name: "scheme1"}}, + }); err != nil { + t.Fatalf("UpdateRepositoryFromGenerate: %v", err) + } + + if _, err := db.Exec(`DROP TABLE colorscheme_groups`); err != nil { + t.Fatalf("drop colorscheme_groups: %v", err) + } + if err := UpdateRepositoryFromGenerate(1, GenerateData{ + Colorschemes: []repository.Colorscheme{{Name: "scheme2", Data: repository.ColorschemeData{ + Light: []repository.ColorschemeGroup{{Name: "Normal", HexCode: "#ffffff"}}, + }}}, + }); err == nil { + t.Fatal("UpdateRepositoryFromGenerate() = nil, want error") + } + + var name string + err := db.QueryRow(`SELECT name FROM colorschemes WHERE repository_id = 1`).Scan(&name) + if err != nil { + t.Fatalf("query row: %v", err) + } + if name != "scheme1" { + t.Fatalf("Name = %q, want %q", name, "scheme1") + } + + var eventCount int + err = db.QueryRow(`SELECT COUNT(*) FROM repository_job_events WHERE repository_id = 1 AND job = 'generate' AND status = 'success'`).Scan(&eventCount) + if err != nil { + t.Fatalf("query row: %v", err) + } + if eventCount != 1 { + t.Fatalf("eventCount = %d, want 1", eventCount) + } + }) + t.Run("creates a generate job event", func(t *testing.T) { setupTestDB(t) insertTestRepo(t, 1, "owner", "repo") diff --git a/internal/database/retry.go b/internal/database/retry.go index e713f66..aec3a7f 100644 --- a/internal/database/retry.go +++ b/internal/database/retry.go @@ -31,7 +31,8 @@ func isTransientLibSQLError(err error) bool { "database is locked", "context deadline exceeded", "too many requests", - "429", + "status=429", + "status 429", "load-shed", "load shed", "rate limit", diff --git a/internal/database/retry_test.go b/internal/database/retry_test.go index 3e8aba3..74ee1ec 100644 --- a/internal/database/retry_test.go +++ b/internal/database/retry_test.go @@ -47,6 +47,11 @@ func TestIsTransientLibSQLError(t *testing.T) { err: testError("rate limit exceeded, retry later"), want: true, }, + { + name: "constraint failure mentioning 429 is not transient", + err: testError("UNIQUE constraint failed: duplicate key 429"), + want: false, + }, { name: "constraint failure is not transient", err: testError("UNIQUE constraint failed"),