From 5d20c66130d6144643120c520f136d8ba75618b0 Mon Sep 17 00:00:00 2001 From: Robin Gagnon Date: Thu, 15 Oct 2026 10:05:26 -0500 Subject: [PATCH 1/2] fix: batch generate group writes each colorscheme group was one remote hrana request, so a large repo sent thousands of requests and tripped turso 429 load-shed. insert groups in multi-row batches of 50 to cut request volume. --- internal/database/repository.go | 117 +++++++++++++++++++-------- internal/database/repository_test.go | 54 +++++++++++++ 2 files changed, 136 insertions(+), 35 deletions(-) diff --git a/internal/database/repository.go b/internal/database/repository.go index 169f2eb..20293c1 100644 --- a/internal/database/repository.go +++ b/internal/database/repository.go @@ -411,10 +411,24 @@ func placeholders(count int) string { return strings.Join(items, ", ") } +// generateGroupBatchSize bounds the rows per colorscheme_groups INSERT. +// Each statement is a remote Hrana request, so batching cuts request volume +// (and 429 load-shed risk). 50 rows x 13 columns = 650 variables, safely +// under SQLite's historic 999-variable limit. +const generateGroupBatchSize = 50 + +const generateGroupColumnCount = 13 + +type generateGroupRow struct { + colorschemeID int64 + background repository.BackgroundValue + group repository.ColorschemeGroup +} + // UpdateRepositoryFromGenerate updates a repository with generate job data. -// The whole transaction retries on transient errors (including Turso 429 -// load-shed). Returns an error so callers can isolate a single repository -// failure instead of aborting the job. +// Group writes are batched and the whole transaction retries on transient +// errors (including Turso 429 load-shed). Returns an error so callers can +// isolate a single repository failure instead of aborting the job. func UpdateRepositoryFromGenerate(id int64, data GenerateData) error { return runRepositoryWriteTransaction(func(ctx context.Context, tx *sql.Tx) error { if _, err := tx.ExecContext(ctx, "DELETE FROM colorschemes WHERE repository_id = ?", id); err != nil { @@ -434,6 +448,7 @@ func UpdateRepositoryFromGenerate(id int64, data GenerateData) error { return fmt.Errorf("update has_dark/has_light: %w", err) } + schemeIDs := make([]int64, 0, len(data.Colorschemes)) for _, scheme := range data.Colorschemes { result, err := tx.ExecContext(ctx, "INSERT INTO colorschemes (repository_id, name) VALUES (?, ?)", id, scheme.Name) if err != nil { @@ -443,6 +458,11 @@ func UpdateRepositoryFromGenerate(id int64, data GenerateData) error { if err != nil { return fmt.Errorf("colorscheme last insert id: %w", err) } + schemeIDs = append(schemeIDs, schemeID) + } + + rows := make([]generateGroupRow, 0) + for index, scheme := range data.Colorschemes { for _, bg := range []struct { value repository.BackgroundValue groups []repository.ColorschemeGroup @@ -451,46 +471,73 @@ func UpdateRepositoryFromGenerate(id int64, data GenerateData) error { {repository.DarkBackground, scheme.Data.Dark}, } { for _, group := range bg.groups { - if _, err := tx.ExecContext(ctx, ` - INSERT INTO colorscheme_groups ( - colorscheme_id, - background, - name, - hex_code, - bold, - italic, - underline, - undercurl, - underdouble, - underdotted, - underdashed, - strikethrough, - reverse - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - schemeID, - bg.value, - group.Name, - group.HexCode, - group.Bold, - group.Italic, - group.Underline, - group.Undercurl, - group.Underdouble, - group.Underdotted, - group.Underdashed, - group.Strikethrough, - group.Reverse, - ); err != nil { - return fmt.Errorf("insert colorscheme group: %w", err) - } + rows = append(rows, generateGroupRow{ + colorschemeID: schemeIDs[index], + background: bg.value, + group: group, + }) } } } + for start := 0; start < len(rows); start += generateGroupBatchSize { + end := min(start+generateGroupBatchSize, len(rows)) + if err := insertGenerateGroupBatch(ctx, tx, rows[start:end]); err != nil { + return err + } + } + return createRepositoryJobEventsContext(ctx, tx, []int64{id}, jobGenerate, jobStatusSuccess, "", time.Now().UTC()) }) } +func insertGenerateGroupBatch(ctx context.Context, tx *sql.Tx, rows []generateGroupRow) error { + if len(rows) == 0 { + return nil + } + + args := make([]any, 0, len(rows)*generateGroupColumnCount) + for _, row := range rows { + args = append(args, + row.colorschemeID, + row.background, + row.group.Name, + row.group.HexCode, + row.group.Bold, + row.group.Italic, + row.group.Underline, + row.group.Undercurl, + row.group.Underdouble, + row.group.Underdotted, + row.group.Underdashed, + row.group.Strikethrough, + row.group.Reverse, + ) + } + + if _, err := tx.ExecContext(ctx, ` + INSERT INTO colorscheme_groups ( + colorscheme_id, + background, + name, + hex_code, + bold, + italic, + underline, + undercurl, + underdouble, + underdotted, + underdashed, + strikethrough, + reverse + ) VALUES `+rowPlaceholders(len(rows), generateGroupColumnCount), + args...); err != nil { + return fmt.Errorf("insert colorscheme groups: %w", err) + } + + return nil +} + type repositoryJobEventExecutor interface { Exec(query string, args ...any) (sql.Result, error) } diff --git a/internal/database/repository_test.go b/internal/database/repository_test.go index 2988f7b..b19670c 100644 --- a/internal/database/repository_test.go +++ b/internal/database/repository_test.go @@ -848,6 +848,60 @@ func TestUpdateRepositoryFromGenerate(t *testing.T) { } }) + t.Run("batches groups spanning multiple batches", func(t *testing.T) { + setupTestDB(t) + insertTestRepo(t, 1, "owner", "repo") + + groupCount := generateGroupBatchSize + 10 + light := make([]repository.ColorschemeGroup, 0, groupCount) + dark := make([]repository.ColorschemeGroup, 0, groupCount) + for index := 0; index < groupCount; index++ { + light = append(light, repository.ColorschemeGroup{ + Name: fmt.Sprintf("light-%d", index), + HexCode: "#ffffff", + Bold: index%2 == 0, + }) + dark = append(dark, repository.ColorschemeGroup{ + Name: fmt.Sprintf("dark-%d", index), + HexCode: "#000000", + Italic: index%2 == 1, + }) + } + + if err := UpdateRepositoryFromGenerate(1, GenerateData{ + Colorschemes: []repository.Colorscheme{ + {Name: "scheme1", Data: repository.ColorschemeData{Light: light}}, + {Name: "scheme2", Data: repository.ColorschemeData{Dark: dark}}, + }, + }); err != nil { + t.Fatalf("UpdateRepositoryFromGenerate: %v", err) + } + + repo, err := GetRepository("owner/repo") + if err != nil { + t.Fatalf("GetRepository: %v", err) + } + if len(repo.Colorschemes) != 2 { + t.Fatalf("Colorschemes len = %d, want 2", len(repo.Colorschemes)) + } + byName := map[string]repository.Colorscheme{} + for _, scheme := range repo.Colorschemes { + byName[scheme.Name] = scheme + } + if len(byName["scheme1"].Data.Light) != groupCount { + t.Fatalf("scheme1 light len = %d, want %d", len(byName["scheme1"].Data.Light), groupCount) + } + if len(byName["scheme2"].Data.Dark) != groupCount { + t.Fatalf("scheme2 dark len = %d, want %d", len(byName["scheme2"].Data.Dark), groupCount) + } + if byName["scheme1"].Data.Light[0].Name != "light-0" { + t.Fatalf("scheme1 light[0] = %q, want %q", byName["scheme1"].Data.Light[0].Name, "light-0") + } + if !byName["scheme1"].Data.Light[0].Bold { + t.Fatal("scheme1 light[0].Bold = false, want true") + } + }) + t.Run("replaces existing colorschemes", func(t *testing.T) { setupTestDB(t) insertTestRepo(t, 1, "owner", "repo") From 88965e8ff507eff0907a54b32b0613b8cf4a98bd Mon Sep 17 00:00:00 2001 From: Robin Gagnon Date: Thu, 15 Oct 2026 10:46:25 -0500 Subject: [PATCH 2/2] Assert background pairing in batch test --- internal/database/repository_test.go | 34 ++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/internal/database/repository_test.go b/internal/database/repository_test.go index b19670c..a8b9157 100644 --- a/internal/database/repository_test.go +++ b/internal/database/repository_test.go @@ -891,15 +891,49 @@ func TestUpdateRepositoryFromGenerate(t *testing.T) { if len(byName["scheme1"].Data.Light) != groupCount { t.Fatalf("scheme1 light len = %d, want %d", len(byName["scheme1"].Data.Light), groupCount) } + if len(byName["scheme1"].Data.Dark) != 0 { + t.Fatalf("scheme1 dark len = %d, want 0", len(byName["scheme1"].Data.Dark)) + } if len(byName["scheme2"].Data.Dark) != groupCount { t.Fatalf("scheme2 dark len = %d, want %d", len(byName["scheme2"].Data.Dark), groupCount) } + if len(byName["scheme2"].Data.Light) != 0 { + t.Fatalf("scheme2 light len = %d, want 0", len(byName["scheme2"].Data.Light)) + } if byName["scheme1"].Data.Light[0].Name != "light-0" { t.Fatalf("scheme1 light[0] = %q, want %q", byName["scheme1"].Data.Light[0].Name, "light-0") } if !byName["scheme1"].Data.Light[0].Bold { t.Fatal("scheme1 light[0].Bold = false, want true") } + if byName["scheme1"].Data.Light[55].Name != "light-55" { + t.Fatalf("scheme1 light[55] = %q, want %q", byName["scheme1"].Data.Light[55].Name, "light-55") + } + if byName["scheme1"].Data.Light[55].Bold { + t.Fatal("scheme1 light[55].Bold = true, want false") + } + if byName["scheme2"].Data.Dark[59].Name != "dark-59" { + t.Fatalf("scheme2 dark[59] = %q, want %q", byName["scheme2"].Data.Dark[59].Name, "dark-59") + } + if !byName["scheme2"].Data.Dark[59].Italic { + t.Fatal("scheme2 dark[59].Italic = false, want true") + } + + var lightCount, darkCount int + err = db.QueryRow(`SELECT COUNT(*) FROM colorscheme_groups WHERE background = 'light'`).Scan(&lightCount) + if err != nil { + t.Fatalf("query row: %v", err) + } + err = db.QueryRow(`SELECT COUNT(*) FROM colorscheme_groups WHERE background = 'dark'`).Scan(&darkCount) + if err != nil { + t.Fatalf("query row: %v", err) + } + if lightCount != groupCount { + t.Fatalf("light background count = %d, want %d", lightCount, groupCount) + } + if darkCount != groupCount { + t.Fatalf("dark background count = %d, want %d", darkCount, groupCount) + } }) t.Run("replaces existing colorschemes", func(t *testing.T) {