Skip to content

fix: batch generate group writes #112

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
reobin merged 2 commits into from
Oct 15, 2026
Merged

fix: batch generate group writes #112

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5d20c66
fix: batch generate group writes
reobin Oct 15, 2026
88965e8
Assert background pairing in batch test
reobin Oct 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 82 additions & 35 deletions internal/database/repository.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -411,10 +411,24 @@ func placeholders(count int) string {
return strings.Join(items, ", ")
}

// generateGroupBatchSize bounds the rows per colorscheme_groups INSERT.
// Each statement is a remote Hrana request, so batching cuts request volume
// (and 429 load-shed risk). 50 rows x 13 columns = 650 variables, safely
// under SQLite's historic 999-variable limit.
const generateGroupBatchSize = 50

const generateGroupColumnCount = 13

type generateGroupRow struct {
colorschemeID int64
background repository.BackgroundValue
group repository.ColorschemeGroup
}

// UpdateRepositoryFromGenerate updates a repository with generate job data.
// The whole transaction retries on transient errors (including Turso 429
// load-shed). Returns an error so callers can isolate a single repository
// failure instead of aborting the job.
// Group writes are batched and the whole transaction retries on transient
// errors (including Turso 429 load-shed). Returns an error so callers can
// isolate a single repository failure instead of aborting the job.
func UpdateRepositoryFromGenerate(id int64, data GenerateData) error {
return runRepositoryWriteTransaction(func(ctx context.Context, tx *sql.Tx) error {
if _, err := tx.ExecContext(ctx, "DELETE FROM colorschemes WHERE repository_id = ?", id); err != nil {
Expand All @@ -434,6 +448,7 @@ func UpdateRepositoryFromGenerate(id int64, data GenerateData) error {
return fmt.Errorf("update has_dark/has_light: %w", err)
}

schemeIDs := make([]int64, 0, len(data.Colorschemes))
for _, scheme := range data.Colorschemes {
result, err := tx.ExecContext(ctx, "INSERT INTO colorschemes (repository_id, name) VALUES (?, ?)", id, scheme.Name)
if err != nil {
Expand All @@ -443,6 +458,11 @@ func UpdateRepositoryFromGenerate(id int64, data GenerateData) error {
if err != nil {
return fmt.Errorf("colorscheme last insert id: %w", err)
}
schemeIDs = append(schemeIDs, schemeID)
}

rows := make([]generateGroupRow, 0)
for index, scheme := range data.Colorschemes {
for _, bg := range []struct {
value repository.BackgroundValue
groups []repository.ColorschemeGroup
Expand All @@ -451,46 +471,73 @@ func UpdateRepositoryFromGenerate(id int64, data GenerateData) error {
{repository.DarkBackground, scheme.Data.Dark},
} {
for _, group := range bg.groups {
if _, err := tx.ExecContext(ctx, `
INSERT INTO colorscheme_groups (
colorscheme_id,
background,
name,
hex_code,
bold,
italic,
underline,
undercurl,
underdouble,
underdotted,
underdashed,
strikethrough,
reverse
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
schemeID,
bg.value,
group.Name,
group.HexCode,
group.Bold,
group.Italic,
group.Underline,
group.Undercurl,
group.Underdouble,
group.Underdotted,
group.Underdashed,
group.Strikethrough,
group.Reverse,
); err != nil {
return fmt.Errorf("insert colorscheme group: %w", err)
}
rows = append(rows, generateGroupRow{
colorschemeID: schemeIDs[index],
background: bg.value,
group: group,
})
}
}
}

for start := 0; start < len(rows); start += generateGroupBatchSize {
end := min(start+generateGroupBatchSize, len(rows))
if err := insertGenerateGroupBatch(ctx, tx, rows[start:end]); err != nil {
return err
}
}

return createRepositoryJobEventsContext(ctx, tx, []int64{id}, jobGenerate, jobStatusSuccess, "", time.Now().UTC())
})
}

func insertGenerateGroupBatch(ctx context.Context, tx *sql.Tx, rows []generateGroupRow) error {
if len(rows) == 0 {
return nil
}

args := make([]any, 0, len(rows)*generateGroupColumnCount)
for _, row := range rows {
args = append(args,
row.colorschemeID,
row.background,
row.group.Name,
row.group.HexCode,
row.group.Bold,
row.group.Italic,
row.group.Underline,
row.group.Undercurl,
row.group.Underdouble,
row.group.Underdotted,
row.group.Underdashed,
row.group.Strikethrough,
row.group.Reverse,
)
}

if _, err := tx.ExecContext(ctx, `
INSERT INTO colorscheme_groups (
colorscheme_id,
background,
name,
hex_code,
bold,
italic,
underline,
undercurl,
underdouble,
underdotted,
underdashed,
strikethrough,
reverse
) VALUES `+rowPlaceholders(len(rows), generateGroupColumnCount),
args...); err != nil {
return fmt.Errorf("insert colorscheme groups: %w", err)
}

return nil
}

type repositoryJobEventExecutor interface {
Exec(query string, args ...any) (sql.Result, error)
}
Expand Down
88 changes: 88 additions & 0 deletions internal/database/repository_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -848,6 +848,94 @@ func TestUpdateRepositoryFromGenerate(t *testing.T) {
}
})

t.Run("batches groups spanning multiple batches", func(t *testing.T) {
setupTestDB(t)
insertTestRepo(t, 1, "owner", "repo")

groupCount := generateGroupBatchSize + 10
light := make([]repository.ColorschemeGroup, 0, groupCount)
dark := make([]repository.ColorschemeGroup, 0, groupCount)
for index := 0; index < groupCount; index++ {
light = append(light, repository.ColorschemeGroup{
Name: fmt.Sprintf("light-%d", index),
HexCode: "#ffffff",
Bold: index%2 == 0,
})
dark = append(dark, repository.ColorschemeGroup{
Name: fmt.Sprintf("dark-%d", index),
HexCode: "#000000",
Italic: index%2 == 1,
})
}

if err := UpdateRepositoryFromGenerate(1, GenerateData{
Colorschemes: []repository.Colorscheme{
{Name: "scheme1", Data: repository.ColorschemeData{Light: light}},
{Name: "scheme2", Data: repository.ColorschemeData{Dark: dark}},
},
}); err != nil {
t.Fatalf("UpdateRepositoryFromGenerate: %v", err)
}

repo, err := GetRepository("owner/repo")
if err != nil {
t.Fatalf("GetRepository: %v", err)
}
if len(repo.Colorschemes) != 2 {
t.Fatalf("Colorschemes len = %d, want 2", len(repo.Colorschemes))
}
byName := map[string]repository.Colorscheme{}
for _, scheme := range repo.Colorschemes {
byName[scheme.Name] = scheme
}
if len(byName["scheme1"].Data.Light) != groupCount {
t.Fatalf("scheme1 light len = %d, want %d", len(byName["scheme1"].Data.Light), groupCount)
}
if len(byName["scheme1"].Data.Dark) != 0 {
t.Fatalf("scheme1 dark len = %d, want 0", len(byName["scheme1"].Data.Dark))
}
if len(byName["scheme2"].Data.Dark) != groupCount {
t.Fatalf("scheme2 dark len = %d, want %d", len(byName["scheme2"].Data.Dark), groupCount)
}
if len(byName["scheme2"].Data.Light) != 0 {
t.Fatalf("scheme2 light len = %d, want 0", len(byName["scheme2"].Data.Light))
}
if byName["scheme1"].Data.Light[0].Name != "light-0" {
t.Fatalf("scheme1 light[0] = %q, want %q", byName["scheme1"].Data.Light[0].Name, "light-0")
}
if !byName["scheme1"].Data.Light[0].Bold {
t.Fatal("scheme1 light[0].Bold = false, want true")
}
if byName["scheme1"].Data.Light[55].Name != "light-55" {
t.Fatalf("scheme1 light[55] = %q, want %q", byName["scheme1"].Data.Light[55].Name, "light-55")
}
if byName["scheme1"].Data.Light[55].Bold {
t.Fatal("scheme1 light[55].Bold = true, want false")
}
if byName["scheme2"].Data.Dark[59].Name != "dark-59" {
t.Fatalf("scheme2 dark[59] = %q, want %q", byName["scheme2"].Data.Dark[59].Name, "dark-59")
}
if !byName["scheme2"].Data.Dark[59].Italic {
t.Fatal("scheme2 dark[59].Italic = false, want true")
}

var lightCount, darkCount int
err = db.QueryRow(`SELECT COUNT(*) FROM colorscheme_groups WHERE background = 'light'`).Scan(&lightCount)
if err != nil {
t.Fatalf("query row: %v", err)
}
err = db.QueryRow(`SELECT COUNT(*) FROM colorscheme_groups WHERE background = 'dark'`).Scan(&darkCount)
if err != nil {
t.Fatalf("query row: %v", err)
}
if lightCount != groupCount {
t.Fatalf("light background count = %d, want %d", lightCount, groupCount)
}
if darkCount != groupCount {
t.Fatalf("dark background count = %d, want %d", darkCount, groupCount)
}
})

t.Run("replaces existing colorschemes", func(t *testing.T) {
setupTestDB(t)
insertTestRepo(t, 1, "owner", "repo")
Expand Down
Loading