From 2afe90513c5c1fad8fb2cc61a442c367544326d9 Mon Sep 17 00:00:00 2001 From: Thierry Schellenbach Date: Thu, 19 Nov 2026 12:53:35 -0700 Subject: [PATCH 1/6] fix(store): give the tool groups migration its own version 20270102090000_agent_config_tool_groups.sql shared its version with 20270102090000_ui_instructions.sql, and goose refuses to run with a duplicate, so the router could not migrate. It is 20270103090000 now. TestARedactedCallStillReportsWhatItAndItsToolCost set redact_at on the global database, so under the sharded suite RedactToolOutput, which runs on the shard, found nothing. Co-authored-by: Cursor --- acceleration/internal/store/models.go | 2 +- acceleration/internal/store/tool_groups_migration_test.go | 6 +++--- acceleration/internal/store/usage_test.go | 2 +- ...oups.sql => 20270103090000_agent_config_tool_groups.sql} | 0 4 files changed, 5 insertions(+), 5 deletions(-) rename acceleration/migrations/{20270102090000_agent_config_tool_groups.sql => 20270103090000_agent_config_tool_groups.sql} (100%) diff --git a/acceleration/internal/store/models.go b/acceleration/internal/store/models.go index 61d8b82a8..cd08f6901 100644 --- a/acceleration/internal/store/models.go +++ b/acceleration/internal/store/models.go @@ -653,7 +653,7 @@ type AgentConfig struct { // EpisodeCards off; WritesCards is the two together. EpisodeCardsRecord bool `bun:"episode_cards_record,notnull"` // ToolGroups are the functions an SDK provides that the config's sessions may offer, and - // how each group is loaded (20270102090000_agent_config_tool_groups.sql). A function no + // how each group is loaded (20270103090000_agent_config_tool_groups.sql). A function no // group names is not offered. The row's progressive_tools and tool_output_retention // columns, which groups and connector bindings replaced, are not modelled: a router still // running the release before reads them, so they are left in place. diff --git a/acceleration/internal/store/tool_groups_migration_test.go b/acceleration/internal/store/tool_groups_migration_test.go index 2b880f02c..6cd918978 100644 --- a/acceleration/internal/store/tool_groups_migration_test.go +++ b/acceleration/internal/store/tool_groups_migration_test.go @@ -15,10 +15,10 @@ import ( "github.com/GetStream/Vision-Agents/acceleration/migrations" ) -// Versions either side of 20270102090000_agent_config_tool_groups.sql. +// Versions either side of 20270103090000_agent_config_tool_groups.sql. const ( - schemaBeforeToolGroups int64 = 20270101090000 - toolGroupsVersion int64 = 20270102090000 + schemaBeforeToolGroups int64 = 20270102090000 + toolGroupsVersion int64 = 20270103090000 ) // ToolGroupsMigrationSuite stops a database of its own short of tool groups, writes configs diff --git a/acceleration/internal/store/usage_test.go b/acceleration/internal/store/usage_test.go index 3d1ae0914..d58873a0f 100644 --- a/acceleration/internal/store/usage_test.go +++ b/acceleration/internal/store/usage_test.go @@ -530,7 +530,7 @@ func (s *StoreSuite) TestARedactedCallStillReportsWhatItAndItsToolCost() { s.calledTool(session, "one-1", "lookup_order", s.base, map[string]any{"call_id": "c1", "loading": ToolLoadingSummary, "definition_tokens": 461, "summary_tokens": 84}, map[string]any{"call_id": "c1", "arguments": `{"order":"A-1"}`, "tokens": 250}) - _, err := s.store.db.NewUpdate().Model((*AgentResponseItem)(nil)). + _, err := s.store.ShardDB().NewUpdate().Model((*AgentResponseItem)(nil)). Set("redact_at = ?", s.base).Where("session_id = ?", "one").Exec(s.ctx) s.Require().NoError(err) redacted, err := s.store.RedactToolOutput(s.ctx, s.base.Add(time.Hour), 100) diff --git a/acceleration/migrations/20270102090000_agent_config_tool_groups.sql b/acceleration/migrations/20270103090000_agent_config_tool_groups.sql similarity index 100% rename from acceleration/migrations/20270102090000_agent_config_tool_groups.sql rename to acceleration/migrations/20270103090000_agent_config_tool_groups.sql From f2005b06d9390f0fe696d626b8df5cfb3a756c7b Mon Sep 17 00:00:00 2001 From: Thierry Schellenbach Date: Thu, 19 Nov 2026 12:56:21 -0700 Subject: [PATCH 2/6] refactor(store): build connector token, attempt and client queries with bun Config tokens, authorization attempts, OAuth clients, the tool re-approval read and the connector delete loop go through bun's builders. The attempt updates share openAttempt's conditions instead of repeating them. Co-authored-by: Cursor --- acceleration/internal/store/config_tokens.go | 38 +++++++------- acceleration/internal/store/connections.go | 52 ++++++++++---------- acceleration/internal/store/connectors.go | 7 +-- acceleration/internal/store/oauth_clients.go | 34 ++++++------- acceleration/internal/store/tool_pins.go | 8 +-- 5 files changed, 71 insertions(+), 68 deletions(-) diff --git a/acceleration/internal/store/config_tokens.go b/acceleration/internal/store/config_tokens.go index 80c0f16e8..974c39da5 100644 --- a/acceleration/internal/store/config_tokens.go +++ b/acceleration/internal/store/config_tokens.go @@ -7,6 +7,8 @@ import ( "fmt" "time" + "github.com/uptrace/bun" + "github.com/GetStream/Vision-Agents/acceleration/internal/stack" ) @@ -23,6 +25,8 @@ var ErrNoConnectorConfigToken = errors.New("store: no configuration token for th // router for one connector, with its refresh token, sealed by the caller as one blob. It is // what creates, updates and deletes the customer's managed provider app. type ConnectorConfigToken struct { + bun.BaseModel `bun:"table:connector_config_tokens,alias:cct"` + CustomerID string `bun:"customer_id,pk"` ConnectorID string `bun:"connector_id,pk"` TokensSealed []byte `bun:"tokens_sealed,notnull"` @@ -62,21 +66,20 @@ func (s *Store) PutConnectorConfigToken(ctx context.Context, token *ConnectorCon ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), credentialDetachedTimeout) defer cancel() now := time.Now().UTC().Truncate(time.Microsecond) - err := s.db.NewRaw(` -INSERT INTO connector_config_tokens AS cct - (customer_id, connector_id, tokens_sealed, kek_version, expires_at, created_at, updated_at) -VALUES (?, ?, ?, ?, ?, ?, ?) -ON CONFLICT (customer_id, connector_id) DO UPDATE -SET tokens_sealed = EXCLUDED.tokens_sealed, - kek_version = EXCLUDED.kek_version, - expires_at = EXCLUDED.expires_at, - updated_at = EXCLUDED.updated_at -RETURNING cct.created_at`, - token.CustomerID, token.ConnectorID, token.TokensSealed, token.KEKVersion, token.ExpiresAt.UTC(), now, now). - Scan(ctx, &token.CreatedAt) + row := *token + row.CreatedAt, row.UpdatedAt = now, now + err := s.db.NewInsert().Model(&row). + On("CONFLICT (customer_id, connector_id) DO UPDATE"). + Set("tokens_sealed = EXCLUDED.tokens_sealed"). + Set("kek_version = EXCLUDED.kek_version"). + Set("expires_at = EXCLUDED.expires_at"). + Set("updated_at = EXCLUDED.updated_at"). + Returning("cct.created_at"). + Scan(ctx) if err != nil { return stack.Wrap(fmt.Errorf("store: put connector config token: %w", err)) } + token.CreatedAt = row.CreatedAt token.UpdatedAt = now return nil } @@ -87,10 +90,9 @@ func (s *Store) ConnectorConfigToken(ctx context.Context, customerID, connectorI return ConnectorConfigToken{}, stack.Wrap(errors.New("store: a customer and a connector id are required")) } var token ConnectorConfigToken - err := s.db.NewRaw(` -SELECT customer_id, connector_id, tokens_sealed, kek_version, expires_at, created_at, updated_at -FROM connector_config_tokens WHERE customer_id = ? AND connector_id = ?`, customerID, connectorID). - Scan(ctx, &token.CustomerID, &token.ConnectorID, &token.TokensSealed, &token.KEKVersion, &token.ExpiresAt, &token.CreatedAt, &token.UpdatedAt) + err := s.db.NewSelect().Model(&token). + Where("cct.customer_id = ? AND cct.connector_id = ?", customerID, connectorID). + Scan(ctx) if errors.Is(err, sql.ErrNoRows) { return ConnectorConfigToken{}, stack.Wrap(fmt.Errorf("%w: %s", ErrNoConnectorConfigToken, connectorID)) } @@ -106,7 +108,9 @@ func (s *Store) DeleteConnectorConfigToken(ctx context.Context, customerID, conn if customerID == "" || connectorID == "" { return stack.Wrap(errors.New("store: a customer and a connector id are required")) } - _, err := s.db.ExecContext(ctx, "DELETE FROM connector_config_tokens WHERE customer_id = ? AND connector_id = ?", customerID, connectorID) + _, err := s.db.NewDelete().Model((*ConnectorConfigToken)(nil)). + Where("customer_id = ? AND connector_id = ?", customerID, connectorID). + Exec(ctx) if err != nil { return stack.Wrap(fmt.Errorf("store: delete connector config token: %w", err)) } diff --git a/acceleration/internal/store/connections.go b/acceleration/internal/store/connections.go index 5ad6418fb..79f522185 100644 --- a/acceleration/internal/store/connections.go +++ b/acceleration/internal/store/connections.go @@ -727,10 +727,10 @@ func (s *Store) ConnectorConnectionReferenced(ctx context.Context, customerID, i return false, stack.Wrap(errors.New("store: a customer and a connection id are required")) } var referenced bool - err := s.db.QueryRowContext(ctx, ` -SELECT `+boundByConfig+` -FROM connector_connections AS cc -WHERE cc.customer_id = ? AND cc.id = ? AND cc.deleted_at IS NULL`, customerID, id).Scan(&referenced) + err := s.db.NewSelect().Model((*ConnectorConnection)(nil)). + ColumnExpr(boundByConfig). + Where("cc.customer_id = ? AND cc.id = ? AND cc.deleted_at IS NULL", customerID, id). + Scan(ctx, &referenced) if errors.Is(err, sql.ErrNoRows) { return false, stack.Wrap(fmt.Errorf("%w: %s", ErrNoConnectorConnection, id)) } @@ -806,15 +806,12 @@ func (s *Store) ConsumeConnectorAuthorizationAttempt(ctx context.Context, state return ConnectorAuthorizationAttempt{}, errors.New("store: an oauth state is required") } var attempt ConnectorAuthorizationAttempt - err := s.db.NewRaw(` -UPDATE connector_authorization_attempts AS caa -SET consumed_at = now() -WHERE caa.state_hash = ? - AND caa.consumed_at IS NULL - AND caa.expires_at > now() - AND EXISTS (SELECT 1 FROM connector_connections AS cc - WHERE cc.id = caa.connection_id AND cc.customer_id = caa.customer_id AND cc.deleted_at IS NULL) -RETURNING caa.*`, AuthorizationStateHash(state)).Scan(ctx, &attempt) + err := s.db.NewUpdate().Model(&attempt). + Set("consumed_at = now()"). + Where("caa.state_hash = ?", AuthorizationStateHash(state)). + ApplyQueryBuilder(attemptOpen). + Returning("caa.*"). + Scan(ctx) if errors.Is(err, sql.ErrNoRows) { return ConnectorAuthorizationAttempt{}, ErrNoAuthorizationAttempt } @@ -833,16 +830,13 @@ func (s *Store) HandOffConnectorAuthorizationAttempt(ctx context.Context, id str if id == "" || len(previous) == 0 || len(sealed) == 0 || kekVersion < 1 { return errors.New("store: a handoff needs an attempt id, the blob it read and a new one under a key version of 1 or more") } - result, err := s.db.NewRaw(` -UPDATE connector_authorization_attempts AS caa -SET attempt_sealed = ?, kek_version = ? -WHERE caa.id = ? - AND caa.attempt_sealed = ? - AND caa.consumed_at IS NULL - AND caa.expires_at > now() - AND EXISTS (SELECT 1 FROM connector_connections AS cc - WHERE cc.id = caa.connection_id AND cc.customer_id = caa.customer_id AND cc.deleted_at IS NULL)`, - sealed, kekVersion, id, previous).Exec(ctx) + result, err := s.db.NewUpdate().Model((*ConnectorAuthorizationAttempt)(nil)). + Set("attempt_sealed = ?", sealed). + Set("kek_version = ?", kekVersion). + Where("caa.id = ?", id). + Where("caa.attempt_sealed = ?", previous). + ApplyQueryBuilder(attemptOpen). + Exec(ctx) if err != nil { return fmt.Errorf("store: hand off authorization attempt: %w", err) } @@ -863,15 +857,21 @@ func AuthorizationStateHash(state string) string { return hex.EncodeToString(hash[:]) } +// attemptOpen keeps the attempts that are unconsumed, unexpired and for a live connection. +func attemptOpen(q bun.QueryBuilder) bun.QueryBuilder { + return q. + Where("caa.consumed_at IS NULL"). + Where("caa.expires_at > now()"). + Where("EXISTS (SELECT 1 FROM connector_connections AS cc WHERE cc.id = caa.connection_id AND cc.customer_id = caa.customer_id AND cc.deleted_at IS NULL)") +} + // openAttempt is the one attempt matching where that is unconsumed, unexpired and for a // live connection. func (s *Store) openAttempt(ctx context.Context, where string, arg any) (ConnectorAuthorizationAttempt, error) { var attempt ConnectorAuthorizationAttempt err := s.db.NewSelect().Model(&attempt). Where(where, arg). - Where("caa.consumed_at IS NULL"). - Where("caa.expires_at > now()"). - Where("EXISTS (SELECT 1 FROM connector_connections AS cc WHERE cc.id = caa.connection_id AND cc.customer_id = caa.customer_id AND cc.deleted_at IS NULL)"). + ApplyQueryBuilder(attemptOpen). Scan(ctx) if errors.Is(err, sql.ErrNoRows) { return ConnectorAuthorizationAttempt{}, ErrNoAuthorizationAttempt diff --git a/acceleration/internal/store/connectors.go b/acceleration/internal/store/connectors.go index 91ad1d8bd..68371eca1 100644 --- a/acceleration/internal/store/connectors.go +++ b/acceleration/internal/store/connectors.go @@ -637,9 +637,10 @@ func (s *Store) DeleteConnectorDefinition(ctx context.Context, customerID, id st } // The deliveries go with their destination (ON DELETE CASCADE, // 20261007040000_connector_event_destinations.sql). - for _, table := range []string{"connector_oauth_clients", "connector_config_tokens", "connector_event_destinations"} { - if _, err := tx.ExecContext(ctx, "DELETE FROM "+table+" WHERE customer_id = ? AND connector_id = ?", customerID, id); err != nil { - return fmt.Errorf("store: delete the connector's %s: %w", table, err) + for _, model := range []any{(*ConnectorOAuthClient)(nil), (*ConnectorConfigToken)(nil), (*EventDestination)(nil)} { + q := tx.NewDelete().Model(model).Where("customer_id = ? AND connector_id = ?", customerID, id) + if _, err := q.Exec(ctx); err != nil { + return fmt.Errorf("store: delete the connector's %s: %w", q.GetTableName(), err) } } _, err = tx.NewDelete().Model((*ConnectorDefinition)(nil)). diff --git a/acceleration/internal/store/oauth_clients.go b/acceleration/internal/store/oauth_clients.go index f9fa226f7..739bc82ca 100644 --- a/acceleration/internal/store/oauth_clients.go +++ b/acceleration/internal/store/oauth_clients.go @@ -114,25 +114,21 @@ func (s *Store) PutConnectorOAuthClient(ctx context.Context, client *ConnectorOA return fmt.Errorf("%w: %s lists %v, not %s", ErrOAuthClientRegistrationNotListed, client.ConnectorID, definition.Manifest.Client.Registration, client.Registration) } - err = tx.NewRaw(` -INSERT INTO connector_oauth_clients AS coc - (customer_id, connector_id, registration, client_id, auth_method, secret_sealed, kek_version, - provider_app_id, stream_app_pk, signing_secret_sealed, signing_kek_version, created_at, updated_at) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) -ON CONFLICT (customer_id, connector_id) DO UPDATE -SET client_id = EXCLUDED.client_id, - auth_method = EXCLUDED.auth_method, - secret_sealed = EXCLUDED.secret_sealed, - kek_version = EXCLUDED.kek_version, - provider_app_id = EXCLUDED.provider_app_id, - signing_secret_sealed = EXCLUDED.signing_secret_sealed, - signing_kek_version = EXCLUDED.signing_kek_version, - updated_at = EXCLUDED.updated_at -WHERE coc.registration = EXCLUDED.registration -RETURNING coc.created_at, coc.stream_app_pk`, - client.CustomerID, client.ConnectorID, client.Registration, client.ClientID, client.AuthMethod, - client.SecretSealed, client.KEKVersion, client.ProviderAppID, nullablePin(client.StreamAppPK), - client.SigningSecretSealed, client.SigningKEKVersion, now, now).Scan(ctx, &createdAt, &pin) + row := *client + row.CreatedAt, row.UpdatedAt = now, now + err = tx.NewInsert().Model(&row). + On("CONFLICT (customer_id, connector_id) DO UPDATE"). + Set("client_id = EXCLUDED.client_id"). + Set("auth_method = EXCLUDED.auth_method"). + Set("secret_sealed = EXCLUDED.secret_sealed"). + Set("kek_version = EXCLUDED.kek_version"). + Set("provider_app_id = EXCLUDED.provider_app_id"). + Set("signing_secret_sealed = EXCLUDED.signing_secret_sealed"). + Set("signing_kek_version = EXCLUDED.signing_kek_version"). + Set("updated_at = EXCLUDED.updated_at"). + Where("coc.registration = EXCLUDED.registration"). + Returning("coc.created_at, coc.stream_app_pk"). + Scan(ctx, &createdAt, &pin) // The conflict's WHERE kept the row, so nothing was written or returned. if errors.Is(err, sql.ErrNoRows) { return fmt.Errorf("%w: %s", ErrOAuthClientRegistration, client.ConnectorID) diff --git a/acceleration/internal/store/tool_pins.go b/acceleration/internal/store/tool_pins.go index 02cced618..47ce48161 100644 --- a/acceleration/internal/store/tool_pins.go +++ b/acceleration/internal/store/tool_pins.go @@ -124,9 +124,11 @@ func (s *Store) ReapproveConnectorTools(ctx context.Context, connection Connecto // FOR SHARE: a consent or a credentials write that begins a new grant updates this row, // so one that comes later waits for this transaction, and one that came first is seen. var sameGrant bool - err := tx.QueryRowContext(ctx, "SELECT connected_at IS NOT DISTINCT FROM ? FROM connector_connections "+ - "WHERE customer_id = ? AND id = ? AND deleted_at IS NULL FOR SHARE", - connection.ConnectedAt, connection.CustomerID, connection.ID).Scan(&sameGrant) + err := tx.NewSelect().Model((*ConnectorConnection)(nil)). + ColumnExpr("cc.connected_at IS NOT DISTINCT FROM ?", connection.ConnectedAt). + Where("cc.customer_id = ? AND cc.id = ? AND cc.deleted_at IS NULL", connection.CustomerID, connection.ID). + For("SHARE"). + Scan(ctx, &sameGrant) if errors.Is(err, sql.ErrNoRows) { return fmt.Errorf("%w: %s", ErrNoConnectorConnection, connection.ID) } From 5c49d86f7ec61b907121ca44c29e9755fcaeae7a Mon Sep 17 00:00:00 2001 From: Thierry Schellenbach Date: Thu, 19 Nov 2026 13:06:19 -0700 Subject: [PATCH 3/6] refactor(store): build inbound queue and event delivery queries with bun The chat and channel inbound queues (all but their claims and the thread lock, whose SQL text tests match), balance notice deliveries and event destination rotation go through bun's builders. The models' jsonb types marshal themselves, so the hand-written json.Marshal and ?::jsonb casts go. Co-authored-by: Cursor --- .../internal/store/balance_notices.go | 19 +++-- .../internal/store/channel_inbound.go | 70 ++++++++----------- acceleration/internal/store/chat_inbound.go | 58 ++++++++------- .../internal/store/event_destinations.go | 22 +++--- 4 files changed, 84 insertions(+), 85 deletions(-) diff --git a/acceleration/internal/store/balance_notices.go b/acceleration/internal/store/balance_notices.go index 77e0a8c26..f61c09dbd 100644 --- a/acceleration/internal/store/balance_notices.go +++ b/acceleration/internal/store/balance_notices.go @@ -2,7 +2,6 @@ package store import ( "context" - "encoding/json" "errors" "fmt" "time" @@ -49,10 +48,6 @@ func (s *Store) NoteBalance(ctx context.Context, notice BalanceNotice, delivery if notice.Kind != NoticeBalanceLow && notice.Kind != NoticeBalanceExhausted { return false, 0, stack.Wrap(fmt.Errorf("store: a balance notice is %s or %s, not %q", NoticeBalanceLow, NoticeBalanceExhausted, notice.Kind)) } - headers, err := json.Marshal(delivery.Headers) - if err != nil { - return false, 0, stack.Wrap(err) - } err = s.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { written, err := tx.NewInsert().Model(¬ice).On("CONFLICT DO NOTHING").Exec(ctx) if err != nil { @@ -80,11 +75,15 @@ func (s *Store) NoteBalance(ctx context.Context, notice BalanceNotice, delivery if err != nil { return err } - if _, err := tx.NewRaw(` -INSERT INTO connector_event_deliveries (destination_id, id, headers, body, attempts, next_attempt_at) -VALUES (?, ?, ?::jsonb, ?, 0, ?) -ON CONFLICT (destination_id, id) DO NOTHING`, - destination.ID, delivery.ID, string(headers), payload, delivery.NextAttemptAt.UTC()).Exec(ctx); err != nil { + row := EventDelivery{ + DestinationID: destination.ID, ID: delivery.ID, Headers: delivery.Headers, Body: payload, + NextAttemptAt: delivery.NextAttemptAt.UTC(), + } + _, err = tx.NewInsert().Model(&row). + Column("destination_id", "id", "headers", "body", "attempts", "next_attempt_at"). + On("CONFLICT (destination_id, id) DO NOTHING"). + Exec(ctx) + if err != nil { return err } queued++ diff --git a/acceleration/internal/store/channel_inbound.go b/acceleration/internal/store/channel_inbound.go index ac9973789..d4c85ce5e 100644 --- a/acceleration/internal/store/channel_inbound.go +++ b/acceleration/internal/store/channel_inbound.go @@ -215,26 +215,12 @@ func queueChannelInbound(ctx context.Context, tx bun.Tx, message ChannelInbound) if _, err := tx.NewRaw("SELECT 1 FROM channel_threads WHERE channel_id = ? FOR NO KEY UPDATE", message.ChannelID).Exec(ctx); err != nil { return false, stack.Wrap(fmt.Errorf("store: queue channel inbound: %w", err)) } - media, err := json.Marshal(message.Media) - if err != nil { - return false, stack.Wrap(err) - } - var forward *string - if message.Forward != nil { - encoded, err := json.Marshal(message.Forward) - if err != nil { - return false, stack.Wrap(err) - } - forward = new(string(encoded)) - } - result, err := tx.NewRaw(` -INSERT INTO channel_thread_inbound (channel_id, provider_message_id, customer_id, connector_id, provider_unit_id, thread_key, - config_id, config_name, author_id, text, media, raw, forward, next_attempt_at) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::jsonb, ?, ?::jsonb, ?) -ON CONFLICT (channel_id, provider_message_id) DO NOTHING`, - message.ChannelID, message.ProviderMessageID, message.CustomerID, message.ConnectorID, message.ProviderUnitID, message.ThreadKey, - message.ConfigID, message.ConfigName, message.AuthorID, message.Text, string(media), message.Raw, forward, - message.NextAttemptAt.UTC()).Exec(ctx) + message.NextAttemptAt = message.NextAttemptAt.UTC() + result, err := tx.NewInsert().Model(&message). + Column("channel_id", "provider_message_id", "customer_id", "connector_id", "provider_unit_id", "thread_key", + "config_id", "config_name", "author_id", "text", "media", "raw", "forward", "next_attempt_at"). + On("CONFLICT (channel_id, provider_message_id) DO NOTHING"). + Exec(ctx) if err != nil { return false, stack.Wrap(fmt.Errorf("store: queue channel inbound: %w", err)) } @@ -309,13 +295,13 @@ func (s *Store) claimChannelInbound(ctx context.Context, query string, now time. // found is false when no message is queued at all. func (s *Store) NextChannelInboundAt(ctx context.Context) (next time.Time, found bool, err error) { var at sql.NullTime - if err := s.db.NewRaw(` -SELECT min(head.next_attempt_at) FROM ( - SELECT DISTINCT ON (channel_id) next_attempt_at - FROM channel_thread_inbound - WHERE state = 'queued' - ORDER BY channel_id, seq -) AS head`).Scan(ctx, &at); err != nil { + head := s.db.NewSelect().Model((*ChannelInbound)(nil)). + DistinctOn("cti.channel_id"). + Column("cti.next_attempt_at"). + Where("cti.state = ?", ChannelInboundQueued). + Order("cti.channel_id", "cti.seq") + err = s.db.NewSelect().TableExpr("(?) AS head", head).ColumnExpr("min(head.next_attempt_at)").Scan(ctx, &at) + if err != nil { return time.Time{}, false, stack.Wrap(fmt.Errorf("store: next channel inbound: %w", err)) } return at.Time, at.Valid, nil @@ -357,8 +343,11 @@ func (s *Store) FailChannelInbound(ctx context.Context, claimed ChannelInbound, if forward(ctx) != nil { return nil } - if _, err := tx.NewRaw("UPDATE channel_thread_inbound SET forward = jsonb_set(forward, '{forwarded}', 'true') WHERE seq = ?", - claimed.Seq).Exec(ctx); err != nil { + _, err = tx.NewUpdate().Model((*ChannelInbound)(nil)). + Set("forward = jsonb_set(forward, '{forwarded}', 'true')"). + Where("seq = ?", claimed.Seq). + Exec(ctx) + if err != nil { return fmt.Errorf("store: fail channel inbound: %w", err) } return nil @@ -383,13 +372,14 @@ func lockChannelInboundDelivery(ctx context.Context, tx bun.Tx, claimed ChannelI // it: one with the same provider id for the delivery, or with none and the same body, as the // event destinations key a forward (eventforward.deliveryID). func channelInboundDeliveryForwarded(ctx context.Context, tx bun.Tx, claimed ChannelInbound) (bool, error) { - var forwarded bool - if err := tx.NewRaw(` -SELECT EXISTS ( - SELECT 1 FROM channel_thread_inbound - WHERE state = 'failed' AND forward->>'forwarded' = 'true' AND customer_id = ? AND connector_id = ? - AND forward->>'id' = ? AND (? <> '' OR raw = ?) -)`, claimed.CustomerID, claimed.ConnectorID, claimed.Forward.ID, claimed.Forward.ID, claimed.Raw).Scan(ctx, &forwarded); err != nil { + forwarded, err := tx.NewSelect().Model((*ChannelInbound)(nil)). + Where("state = ?", ChannelInboundFailed). + Where("forward->>'forwarded' = 'true'"). + Where("customer_id = ? AND connector_id = ?", claimed.CustomerID, claimed.ConnectorID). + Where("forward->>'id' = ?", claimed.Forward.ID). + Where("(? <> '' OR raw = ?)", claimed.Forward.ID, claimed.Raw). + Exists(ctx) + if err != nil { return false, fmt.Errorf("store: fail channel inbound: %w", err) } return forwarded, nil @@ -452,10 +442,10 @@ func updateClaimedChannelInbound(ctx context.Context, claimed ChannelInbound, up // PruneChannelInbound deletes the rows of written messages finished a day before now, and of // given-up ones 30 days before, and returns how many it deleted. func (s *Store) PruneChannelInbound(ctx context.Context, now time.Time) (int, error) { - result, err := s.db.NewRaw(` -DELETE FROM channel_thread_inbound -WHERE (state = 'done' AND finished_at < ?) OR (state = 'failed' AND finished_at < ?)`, - now.Add(-channelInboundDoneKeep).UTC(), now.Add(-channelInboundFailedKeep).UTC()).Exec(ctx) + result, err := s.db.NewDelete().Model((*ChannelInbound)(nil)). + Where("(state = ? AND finished_at < ?) OR (state = ? AND finished_at < ?)", + ChannelInboundDone, now.Add(-channelInboundDoneKeep).UTC(), ChannelInboundFailed, now.Add(-channelInboundFailedKeep).UTC()). + Exec(ctx) if err != nil { return 0, stack.Wrap(fmt.Errorf("store: prune channel inbound: %w", err)) } diff --git a/acceleration/internal/store/chat_inbound.go b/acceleration/internal/store/chat_inbound.go index 36e1ab824..142e264bc 100644 --- a/acceleration/internal/store/chat_inbound.go +++ b/acceleration/internal/store/chat_inbound.go @@ -82,12 +82,11 @@ func queueChatInbound(ctx context.Context, db bun.IDB, delivery ChatInbound) (bo if delivery.StreamApp == 0 { return false, stack.Wrap(errors.New("store: a Stream app is required: a message queued without one would be queued again once the app is known")) } - result, err := db.NewRaw(` -INSERT INTO chat_inbound (stream_app, message_id, customer_id, deployment, api_key, body, next_attempt_at) -VALUES (?, ?, ?, ?, ?, ?, ?) -ON CONFLICT (stream_app, message_id) DO NOTHING`, - delivery.StreamApp, delivery.MessageID, delivery.CustomerID, delivery.Deployment, delivery.APIKey, delivery.Body, - delivery.NextAttemptAt.UTC()).Exec(ctx) + delivery.NextAttemptAt = delivery.NextAttemptAt.UTC() + result, err := db.NewInsert().Model(&delivery). + Column("stream_app", "message_id", "customer_id", "deployment", "api_key", "body", "next_attempt_at"). + On("CONFLICT (stream_app, message_id) DO NOTHING"). + Exec(ctx) if err != nil { return false, stack.Wrap(fmt.Errorf("store: queue chat inbound: %w", err)) } @@ -164,7 +163,7 @@ func (s *Store) claimChatInbound(ctx context.Context, query string, now time.Tim // ChatInboundBySeq returns the message queued as seq, in whatever state it is. func (s *Store) ChatInboundBySeq(ctx context.Context, seq int64) (ChatInbound, error) { var delivery ChatInbound - err := s.db.NewRaw("SELECT "+chatInboundColumns+" FROM chat_inbound AS cd WHERE cd.seq = ?", seq).Scan(ctx, &delivery) + err := s.db.NewSelect().Model(&delivery).Where("cd.seq = ?", seq).Scan(ctx) if errors.Is(err, sql.ErrNoRows) { return ChatInbound{}, stack.Wrap(fmt.Errorf("%w: %d", ErrNoChatInbound, seq)) } @@ -178,7 +177,11 @@ func (s *Store) ChatInboundBySeq(ctx context.Context, seq int64) (ChatInbound, e // their lease. found is false when no message is queued at all. func (s *Store) NextChatInboundAt(ctx context.Context) (next time.Time, found bool, err error) { var at sql.NullTime - if err := s.db.NewRaw("SELECT min(next_attempt_at) FROM chat_inbound WHERE state = 'queued'").Scan(ctx, &at); err != nil { + err = s.db.NewSelect().Model((*ChatInbound)(nil)). + ColumnExpr("min(next_attempt_at)"). + Where("state = ?", ChatInboundQueued). + Scan(ctx, &at) + if err != nil { return time.Time{}, false, stack.Wrap(fmt.Errorf("store: next chat inbound: %w", err)) } return at.Time, at.Valid, nil @@ -239,10 +242,12 @@ func updateClaimedChatInbound(ctx context.Context, claimed ChatInbound, states [ // nothing is queued under streamApp and messageID: there is nothing to answer then. The lease // is left as it is, for the worker holding it to finish the row (FinishChatInbound). func (s *Store) StartChatInboundAnswer(ctx context.Context, streamApp int64, messageID string, at time.Time) (started bool, err error) { - result, err := s.db.NewRaw(` -UPDATE chat_inbound SET state = 'done', finished_at = ? -WHERE stream_app = ? AND message_id = ? AND state = 'queued'`, - at.UTC(), streamApp, messageID).Exec(ctx) + result, err := s.db.NewUpdate().Model((*ChatInbound)(nil)). + Set("state = ?", ChatInboundDone). + Set("finished_at = ?", at.UTC()). + Where("stream_app = ? AND message_id = ?", streamApp, messageID). + Where("state = ?", ChatInboundQueued). + Exec(ctx) if err != nil { return false, stack.Wrap(fmt.Errorf("store: start chat inbound answer: %w", err)) } @@ -257,10 +262,12 @@ WHERE stream_app = ? AND message_id = ? AND state = 'queued'`, // messageID said, so a later try of it hands that on rather than hearing it again. What was // kept first stays. func (s *Store) KeepChatInboundHeard(ctx context.Context, streamApp int64, messageID, heard string) error { - if _, err := s.db.NewRaw(` -UPDATE chat_inbound SET heard = ? -WHERE stream_app = ? AND message_id = ? AND heard IS NULL`, - heard, streamApp, messageID).Exec(ctx); err != nil { + _, err := s.db.NewUpdate().Model((*ChatInbound)(nil)). + Set("heard = ?", heard). + Where("stream_app = ? AND message_id = ?", streamApp, messageID). + Where("heard IS NULL"). + Exec(ctx) + if err != nil { return stack.Wrap(fmt.Errorf("store: keep chat inbound heard: %w", err)) } return nil @@ -270,10 +277,13 @@ WHERE stream_app = ? AND message_id = ? AND heard IS NULL`, // and that nothing then took, such as a dispatch worker that was not connected, so a later try // answers it. The lease is left as it is, for the worker holding it to retry the row. func (s *Store) ReleaseChatInboundAnswer(ctx context.Context, streamApp int64, messageID string) error { - if _, err := s.db.NewRaw(` -UPDATE chat_inbound SET state = 'queued', finished_at = NULL -WHERE stream_app = ? AND message_id = ? AND state = 'done'`, - streamApp, messageID).Exec(ctx); err != nil { + _, err := s.db.NewUpdate().Model((*ChatInbound)(nil)). + Set("state = ?", ChatInboundQueued). + Set("finished_at = NULL"). + Where("stream_app = ? AND message_id = ?", streamApp, messageID). + Where("state = ?", ChatInboundDone). + Exec(ctx) + if err != nil { return stack.Wrap(fmt.Errorf("store: release chat inbound answer: %w", err)) } return nil @@ -282,10 +292,10 @@ WHERE stream_app = ? AND message_id = ? AND state = 'done'`, // PruneChatInbound deletes the rows of answered messages finished a day before now, and of // given-up ones 30 days before, and returns how many it deleted. func (s *Store) PruneChatInbound(ctx context.Context, now time.Time) (int, error) { - result, err := s.db.NewRaw(` -DELETE FROM chat_inbound -WHERE (state = 'done' AND finished_at < ?) OR (state = 'failed' AND finished_at < ?)`, - now.Add(-chatInboundDoneKeep).UTC(), now.Add(-chatInboundFailedKeep).UTC()).Exec(ctx) + result, err := s.db.NewDelete().Model((*ChatInbound)(nil)). + Where("(state = ? AND finished_at < ?) OR (state = ? AND finished_at < ?)", + ChatInboundDone, now.Add(-chatInboundDoneKeep).UTC(), ChatInboundFailed, now.Add(-chatInboundFailedKeep).UTC()). + Exec(ctx) if err != nil { return 0, stack.Wrap(fmt.Errorf("store: prune chat inbound: %w", err)) } diff --git a/acceleration/internal/store/event_destinations.go b/acceleration/internal/store/event_destinations.go index af2c6c4e3..a32f5dea7 100644 --- a/acceleration/internal/store/event_destinations.go +++ b/acceleration/internal/store/event_destinations.go @@ -162,16 +162,16 @@ func (s *Store) RotateEventDestinationSecret(ctx context.Context, customerID, co } now := time.Now().UTC().Truncate(time.Microsecond) var destination EventDestination - err := s.db.NewRaw(` -UPDATE connector_event_destinations -SET previous_secret_sealed = secret_sealed, - previous_kek_version = kek_version, - previous_until = ?, - secret_sealed = ?, - kek_version = ?, - updated_at = ? -WHERE id = ? AND customer_id = ? AND connector_id = ? -RETURNING *`, previousUntil.UTC(), sealed, version, now, id, customerID, connectorID).Scan(ctx, &destination) + err := s.db.NewUpdate().Model(&destination). + Set("previous_secret_sealed = secret_sealed"). + Set("previous_kek_version = kek_version"). + Set("previous_until = ?", previousUntil.UTC()). + Set("secret_sealed = ?", sealed). + Set("kek_version = ?", version). + Set("updated_at = ?", now). + Where("id = ? AND customer_id = ? AND connector_id = ?", id, customerID, connectorID). + Returning("*"). + Scan(ctx) if errors.Is(err, sql.ErrNoRows) { return EventDestination{}, stack.Wrap(fmt.Errorf("%w: %s", ErrNoEventDestination, id)) } @@ -316,7 +316,7 @@ func (s *Store) claimEventDeliveries(ctx context.Context, query string, now time // end of their lease. found is false when no delivery is queued at all. func (s *Store) NextEventDeliveryAt(ctx context.Context) (next time.Time, found bool, err error) { var at sql.NullTime - if err := s.db.NewRaw("SELECT min(next_attempt_at) FROM connector_event_deliveries").Scan(ctx, &at); err != nil { + if err := s.db.NewSelect().Model((*EventDelivery)(nil)).ColumnExpr("min(next_attempt_at)").Scan(ctx, &at); err != nil { return time.Time{}, false, stack.Wrap(fmt.Errorf("store: next event delivery: %w", err)) } return at.Time, at.Valid, nil From dc8530d9ba6514a2ab1ebe062bab9e2a79e3e780 Mon Sep 17 00:00:00 2001 From: Thierry Schellenbach Date: Thu, 19 Nov 2026 13:08:07 -0700 Subject: [PATCH 4/6] refactor(store): build image job, reship and campaign lease claims with bun ClaimImageJobs, Reship, Unshipped, TakeOverExpired and PutOffClassification go through bun's With, TableExpr and For. Reship and Unshipped had no test; they have now, and the tests pass on the raw SQL they replace too. Co-authored-by: Cursor --- .../internal/store/campaign_dispatch.go | 36 ++++++---- acceleration/internal/store/image_jobs.go | 27 +++---- acceleration/internal/store/shipping.go | 39 +++++++---- acceleration/internal/store/shipping_test.go | 70 +++++++++++++++++++ 4 files changed, 129 insertions(+), 43 deletions(-) diff --git a/acceleration/internal/store/campaign_dispatch.go b/acceleration/internal/store/campaign_dispatch.go index f89383d1f..e777da587 100644 --- a/acceleration/internal/store/campaign_dispatch.go +++ b/acceleration/internal/store/campaign_dispatch.go @@ -140,18 +140,21 @@ func (s *Store) TakeOverExpired(ctx context.Context, owner string, now time.Time return nil, nil } + expired := s.db.NewSelect().Model((*Contact)(nil)). + Column("cc.id"). + Where("cc.stage = ? AND cc.leased_until < ?", InProgress, now). + Where("NOT EXISTS (SELECT 1 FROM campaigns cp WHERE cp.id = cc.campaign_id AND cp.move_paused)"). + Order("cc.leased_until"). + Limit(limit). + For("UPDATE SKIP LOCKED") var contacts []Contact - err := s.db.NewRaw(`WITH expired AS ( - SELECT id FROM campaign_contacts cc - WHERE stage = ? AND leased_until < ? - AND NOT EXISTS (SELECT 1 FROM campaigns cp WHERE cp.id = cc.campaign_id AND cp.move_paused) - ORDER BY leased_until - LIMIT ? - FOR UPDATE SKIP LOCKED -) -UPDATE campaign_contacts cc SET leased_by = ?, leased_until = ? -FROM expired WHERE cc.id = expired.id -RETURNING cc.*`, InProgress, now, limit, owner, now.Add(lease)).Scan(ctx, &contacts) + err := s.db.NewUpdate().With("expired", expired).Model((*Contact)(nil)). + TableExpr("expired"). + Set("leased_by = ?", owner). + Set("leased_until = ?", now.Add(lease)). + Where("cc.id = expired.id"). + Returning("cc.*"). + Scan(ctx, &contacts) if err != nil { return nil, stack.Wrap(fmt.Errorf("store: take over expired leases: %w", err)) } @@ -457,10 +460,13 @@ func (s *Store) PutOffClassification(ctx context.Context, owner, attemptID strin } var released int64 err := s.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { - result, err := tx.ExecContext(ctx, `UPDATE campaign_contacts cc SET leased_by = '', leased_until = NULL -FROM campaign_attempts ca -WHERE ca.id = ? AND ca.classify_state = ? AND cc.id = ca.contact_id AND cc.stage = ? AND cc.leased_by = ?`, - attemptID, classifyPending, AwaitingOutcome, owner) + result, err := tx.NewUpdate().Model((*Contact)(nil)). + TableExpr("campaign_attempts AS ca"). + Set("leased_by = ''"). + Set("leased_until = NULL"). + Where("ca.id = ? AND ca.classify_state = ?", attemptID, classifyPending). + Where("cc.id = ca.contact_id AND cc.stage = ? AND cc.leased_by = ?", AwaitingOutcome, owner). + Exec(ctx) if err != nil { return fmt.Errorf("store: release a contact to classify later: %w", err) } diff --git a/acceleration/internal/store/image_jobs.go b/acceleration/internal/store/image_jobs.go index c2057453d..039e253f3 100644 --- a/acceleration/internal/store/image_jobs.go +++ b/acceleration/internal/store/image_jobs.go @@ -122,19 +122,20 @@ func (s *Store) ClaimImageJobs(ctx context.Context, now time.Time, limit int, le if limit < 1 { return claimed, nil } - err := s.db.NewRaw(` -WITH due AS ( - SELECT ij.id FROM image_jobs AS ij - WHERE ij.state = 'queued' AND ij.next_attempt_at <= ? AND (ij.message <> '') = ? - ORDER BY ij.next_attempt_at, ij.id - LIMIT ? - FOR UPDATE OF ij SKIP LOCKED -) -UPDATE image_jobs AS ij -SET next_attempt_at = ?, attempts = ij.attempts + 1 -FROM due -WHERE ij.id = due.id -RETURNING `+imageJobClaimed, now.UTC(), decided, limit, leaseUntil.UTC()).Scan(ctx, &claimed) + due := s.db.NewSelect().Model((*ImageJob)(nil)). + Column("ij.id"). + Where("ij.state = 'queued' AND ij.next_attempt_at <= ?", now.UTC()). + Where("(ij.message <> '') = ?", decided). + Order("ij.next_attempt_at", "ij.id"). + Limit(limit). + For("UPDATE OF ij SKIP LOCKED") + err := s.db.NewUpdate().With("due", due).Model((*ImageJob)(nil)). + TableExpr("due"). + Set("next_attempt_at = ?", leaseUntil.UTC()). + Set("attempts = ij.attempts + 1"). + Where("ij.id = due.id"). + Returning(imageJobClaimed). + Scan(ctx, &claimed) if err != nil { return nil, stack.Wrap(fmt.Errorf("store: claim image jobs: %w", err)) } diff --git a/acceleration/internal/store/shipping.go b/acceleration/internal/store/shipping.go index 7dfc110f0..1971d5eb2 100644 --- a/acceleration/internal/store/shipping.go +++ b/acceleration/internal/store/shipping.go @@ -142,9 +142,12 @@ func boundShipping(ctx context.Context, tx bun.Tx) error { func (s *Store) Unshipped(ctx context.Context) (int64, time.Duration, error) { var rows int64 var oldest float64 - err := s.shard.QueryRowContext(ctx, - "SELECT count(*), COALESCE(EXTRACT(EPOCH FROM now() - min(started_at)), 0)::float8 FROM requests WHERE to_ship AND organization_id <> ''", - ).Scan(&rows, &oldest) + err := s.shard.NewSelect().Model((*Request)(nil)). + ColumnExpr("count(*)"). + ColumnExpr("COALESCE(EXTRACT(EPOCH FROM now() - min(r.started_at)), 0)::float8"). + Where("r.to_ship"). + Where("r.organization_id <> ''"). + Scan(ctx, &rows, &oldest) if err != nil { return 0, 0, stack.Wrap(fmt.Errorf("store: unshipped: %w", err)) } @@ -168,21 +171,27 @@ func (s *Store) Reship(ctx context.Context, since time.Time, rejected bool) (int if !last.Valid { return 0, nil } - filter, args := "shipped_at IS NOT NULL AND shipped_at < ?", []any{started} - if rejected { - filter, args = "ship_error <> ''", nil - } var queued int64 for { + batch := s.shard.NewSelect().Model((*Request)(nil)). + Column("r.id"). + Where("r.id > ? AND r.started_at >= ?", last.Int64, since). + Where("r.usage_id IS NOT NULL AND NOT r.to_ship"). + Order("r.id"). + Limit(reshipBatch) + if rejected { + batch.Where("r.ship_error <> ''") + } else { + batch.Where("r.shipped_at IS NOT NULL AND r.shipped_at < ?", started) + } var ids []int64 - err := s.shard.NewRaw(` -WITH batch AS ( - SELECT id FROM requests - WHERE id > ? AND started_at >= ? AND usage_id IS NOT NULL AND NOT to_ship AND `+filter+` - ORDER BY id LIMIT ? -) -UPDATE requests AS r SET to_ship = true, ship_error = '' FROM batch WHERE r.id = batch.id -RETURNING r.id`, append(append([]any{last.Int64, since}, args...), reshipBatch)...).Scan(ctx, &ids) + err := s.shard.NewUpdate().With("batch", batch).Model((*Request)(nil)). + TableExpr("batch"). + Set("to_ship = true"). + Set("ship_error = ''"). + Where("r.id = batch.id"). + Returning("r.id"). + Scan(ctx, &ids) if err != nil { return queued, stack.Wrap(fmt.Errorf("store: reship: %w", err)) } diff --git a/acceleration/internal/store/shipping_test.go b/acceleration/internal/store/shipping_test.go index 123f0a705..036e89f63 100644 --- a/acceleration/internal/store/shipping_test.go +++ b/acceleration/internal/store/shipping_test.go @@ -29,6 +29,76 @@ func (s *StoreSuite) TestMarkingARowAnotherSessionHoldsFailsWithinTheLockWait() s.ErrorContains(err, "lock timeout", "the sweeper's other write waits no longer") } +// shippable records a row of organization's started at at, still to be shipped. +func (s *StoreSuite) shippable(organization string, at time.Time) uuid.UUID { + request := &Request{UsageID: uuid.Must(uuid.NewV7()), Modality: "llm", CustomerID: "acme", OrganizationID: organization, + Provider: "openai", Model: "gpt", StartedAt: at, Success: true, ToShip: true} + s.Require().NoError(s.store.RecordRequest(s.ctx, request)) + return request.UsageID +} + +// shipping is the row's to_ship and ship_error. +func (s *StoreSuite) shipping(usageID uuid.UUID) (bool, string) { + var request Request + s.Require().NoError(s.store.ShardDB().NewSelect().Model(&request).Where("usage_id = ?", usageID).Scan(s.ctx)) + return request.ToShip, request.ShipError +} + +func (s *StoreSuite) TestUnshippedCountsTheRowsWithAnOrganizationStillToShip() { + now := time.Now().UTC() + s.shippable("org", now.Add(-2*time.Hour)) + s.shippable("org", now.Add(-time.Hour)) + s.shippable("", now.Add(-3*time.Hour)) + shipped := s.shippable("org", now.Add(-4*time.Hour)) + s.Require().NoError(s.store.RecordShipment(s.ctx, []uuid.UUID{shipped}, nil)) + + rows, oldest, err := s.store.Unshipped(s.ctx) + + s.Require().NoError(err) + s.EqualValues(2, rows, "neither the shipped row nor the one with no organization") + s.InDelta((2 * time.Hour).Seconds(), oldest.Seconds(), 60) +} + +func (s *StoreSuite) TestReshipQueuesTheShippedRowsStartedSinceAgain() { + since := time.Now().UTC().Add(-time.Hour) + recent := s.shippable("org", since.Add(time.Minute)) + older := s.shippable("org", since.Add(-time.Minute)) + refused := s.shippable("org", since.Add(2*time.Minute)) + waiting := s.shippable("org", since.Add(3*time.Minute)) + s.Require().NoError(s.store.RecordShipment(s.ctx, []uuid.UUID{recent, older}, map[uuid.UUID]string{refused: "unknown price"})) + + queued, err := s.store.Reship(s.ctx, since, false) + + s.Require().NoError(err) + s.EqualValues(1, queued) + toShip, _ := s.shipping(recent) + s.True(toShip) + toShip, _ = s.shipping(older) + s.False(toShip, "started before since") + toShip, reason := s.shipping(refused) + s.False(toShip, "the service refused it, it never had it") + s.Equal("unknown price", reason) + toShip, _ = s.shipping(waiting) + s.True(toShip, "still queued, as it was") +} + +func (s *StoreSuite) TestReshipOfRejectedRowsQueuesTheRefusedOnesAndClearsWhy() { + since := time.Now().UTC().Add(-time.Hour) + shipped := s.shippable("org", since.Add(time.Minute)) + refused := s.shippable("org", since.Add(2*time.Minute)) + s.Require().NoError(s.store.RecordShipment(s.ctx, []uuid.UUID{shipped}, map[uuid.UUID]string{refused: "unknown price"})) + + queued, err := s.store.Reship(s.ctx, since, true) + + s.Require().NoError(err) + s.EqualValues(1, queued) + toShip, reason := s.shipping(refused) + s.True(toShip) + s.Empty(reason) + toShip, _ = s.shipping(shipped) + s.False(toShip, "the service has it") +} + func (s *StoreSuite) TestPostgresProbesTheConnectionHoldingTheShipLock() { lock := s.store.ShipLock() defer lock.Release() From b010bf49bc74cd1b632edab90953eddefea7b7bd Mon Sep 17 00:00:00 2001 From: Thierry Schellenbach Date: Thu, 19 Nov 2026 13:16:48 -0700 Subject: [PATCH 5/6] refactor(store): build reads, forgets and counts on existing models with bun User organizations and settings, linked phones, active price entries, organization apps, tag keys, turns, grant locks, episode summaries, campaign counts and the ledger's forget and prune go through bun's builders. qualified() goes: bun names a model's columns itself. reconcileGrantsQuery stays raw: a test EXPLAINs its text. OrganizationApps had no test; it has now, passing on the old statement too. Co-authored-by: Cursor --- .../internal/store/campaign_analytics.go | 26 +++++++--- acceleration/internal/store/episodes.go | 22 ++++---- acceleration/internal/store/forget.go | 36 +++++++------ acceleration/internal/store/grants.go | 11 ++-- acceleration/internal/store/policies_test.go | 23 ++++++++ acceleration/internal/store/price_changes.go | 8 +-- acceleration/internal/store/prices.go | 23 +++----- acceleration/internal/store/store.go | 52 +++++++++++-------- acceleration/internal/store/user_phones.go | 30 +++++++---- acceleration/internal/usage/ledger/forget.go | 32 +++++++++--- acceleration/internal/usage/ledger/ledger.go | 4 +- 11 files changed, 165 insertions(+), 102 deletions(-) diff --git a/acceleration/internal/store/campaign_analytics.go b/acceleration/internal/store/campaign_analytics.go index 20b08ffb9..7ae6306ab 100644 --- a/acceleration/internal/store/campaign_analytics.go +++ b/acceleration/internal/store/campaign_analytics.go @@ -72,8 +72,12 @@ WHERE cc.campaign_id = ? AND EXISTS (SELECT 1 FROM campaign_attempts ca Outcome string Count int } - err = s.db.NewRaw(`SELECT outcome, count(*) AS count FROM campaign_contacts -WHERE campaign_id = ? AND stage = ? GROUP BY outcome`, campaignID, ContactCompleted).Scan(ctx, &outcomes) + err = s.db.NewSelect().Model((*Contact)(nil)). + Column("cc.outcome"). + ColumnExpr("count(*) AS count"). + Where("cc.campaign_id = ? AND cc.stage = ?", campaignID, ContactCompleted). + Group("cc.outcome"). + Scan(ctx, &outcomes) if err != nil { return CampaignAnalytics{}, stack.Wrap(fmt.Errorf("store: count outcomes: %w", err)) } @@ -85,8 +89,12 @@ WHERE campaign_id = ? AND stage = ? GROUP BY outcome`, campaignID, ContactComple CallOutcome string Count int } - err = s.db.NewRaw(`SELECT call_outcome, count(*) AS count FROM campaign_attempts -WHERE campaign_id = ? AND call_outcome IS NOT NULL GROUP BY call_outcome`, campaignID).Scan(ctx, &calls) + err = s.db.NewSelect().Model((*CampaignAttempt)(nil)). + Column("ca.call_outcome"). + ColumnExpr("count(*) AS count"). + Where("ca.campaign_id = ? AND ca.call_outcome IS NOT NULL", campaignID). + Group("ca.call_outcome"). + Scan(ctx, &calls) if err != nil { return CampaignAnalytics{}, stack.Wrap(fmt.Errorf("store: count call outcomes: %w", err)) } @@ -119,9 +127,13 @@ func (s *Store) ContactStages(ctx context.Context, campaignIDs []string) (map[st Stage string Count int } - err := s.db.NewRaw(`SELECT campaign_id, CASE WHEN stage = ? THEN ? ELSE stage END AS stage, count(*) AS count -FROM campaign_contacts WHERE campaign_id IN (?) GROUP BY 1, 2`, - AwaitingOutcome, InProgress, bun.In(campaignIDs)).Scan(ctx, &rows) + err := s.db.NewSelect().Model((*Contact)(nil)). + Column("cc.campaign_id"). + ColumnExpr("CASE WHEN cc.stage = ? THEN ? ELSE cc.stage END AS stage", AwaitingOutcome, InProgress). + ColumnExpr("count(*) AS count"). + Where("cc.campaign_id IN (?)", bun.In(campaignIDs)). + GroupExpr("1, 2"). + Scan(ctx, &rows) if err != nil { return nil, stack.Wrap(fmt.Errorf("store: count contact stages: %w", err)) } diff --git a/acceleration/internal/store/episodes.go b/acceleration/internal/store/episodes.go index ca37ef86b..159cd8931 100644 --- a/acceleration/internal/store/episodes.go +++ b/acceleration/internal/store/episodes.go @@ -309,24 +309,22 @@ func (s *Store) ClaimEpisodeSummaries(ctx context.Context, now time.Time, limit return claimed, nil } -// finishEpisodeSummaryQuery sets an ended episode's status and drops its lease, in one -// statement. -const finishEpisodeSummaryQuery = ` -WITH finished AS ( - UPDATE episodes SET status = ? - WHERE customer_id = ? AND id = ? AND status = 'ended' - RETURNING id -) -UPDATE episode_activity AS act SET summary_lease_until = NULL -FROM finished WHERE act.episode_id = finished.id` - // FinishEpisodeSummary sets an ended episode summarized or summary_failed and drops its // lease. An episode no longer ended, finished already, is left as it is. func (s *Store) FinishEpisodeSummary(ctx context.Context, customerID, id, status string) error { if status != EpisodeSummarized && status != EpisodeSummaryFailed { return stack.Wrap(fmt.Errorf("store: %q is not a status a summary finishes with", status)) } - _, err := s.db.NewRaw(finishEpisodeSummaryQuery, status, customerID, id).Exec(ctx) + finished := s.db.NewUpdate().Model((*Episode)(nil)). + Set("status = ?", status). + Where("ep.customer_id = ? AND ep.id = ? AND ep.status = ?", customerID, id, EpisodeEnded). + Returning("ep.id") + _, err := s.db.NewUpdate().With("finished", finished). + TableExpr("episode_activity AS act"). + TableExpr("finished"). + Set("summary_lease_until = NULL"). + Where("act.episode_id = finished.id"). + Exec(ctx) if err != nil { return stack.Wrap(fmt.Errorf("store: finish episode summary: %w", err)) } diff --git a/acceleration/internal/store/forget.go b/acceleration/internal/store/forget.go index dfd0ebe5a..d59832605 100644 --- a/acceleration/internal/store/forget.go +++ b/acceleration/internal/store/forget.go @@ -66,8 +66,11 @@ func (s *Store) RenameUserRequests(ctx context.Context, appID, userID, replaceme func (s *Store) UserOrganizations(ctx context.Context, appID string, userIDs ...string) ([]string, error) { shards, err := everyShard(ctx, s, func(ctx context.Context, db bun.IDB) ([]string, error) { var found []string - err := db.NewRaw(`SELECT DISTINCT organization_id FROM requests WHERE customer_id = ? AND user_id IN (?)`, - appID, bun.In(userIDs)).Scan(ctx, &found) + err := db.NewSelect().Model((*Request)(nil)). + Distinct(). + Column("r.organization_id"). + Where("r.customer_id = ? AND r.user_id IN (?)", appID, bun.In(userIDs)). + Scan(ctx, &found) return found, err }) if err != nil { @@ -75,12 +78,14 @@ func (s *Store) UserOrganizations(ctx context.Context, appID string, userIDs ... } var shared []string users := bun.In(userIDs) - if err := s.db.NewRaw(`SELECT organization_id FROM app_organizations WHERE app_id = ? - UNION SELECT organization_id FROM apps WHERE id = ? - UNION SELECT organization_id FROM limits WHERE level = ? AND app_id = ? AND user_id IN (?) - UNION SELECT organization_id FROM usage_resets WHERE app_id = ? AND user_id IN (?) - UNION SELECT organization_id FROM limit_grants WHERE app_id = ? AND user_id IN (?)`, - appID, appID, LevelUser, appID, users, appID, users, appID, users).Scan(ctx, &shared); err != nil { + err = s.db.NewSelect().Model((*AppOrganization)(nil)).Column("organization_id").Where("app_id = ?", appID). + Union(s.db.NewSelect().Model((*App)(nil)).Column("organization_id").Where("id = ?", appID)). + Union(s.db.NewSelect().Model((*Limit)(nil)).Column("organization_id"). + Where("level = ? AND app_id = ? AND user_id IN (?)", LevelUser, appID, users)). + Union(s.db.NewSelect().Model((*UsageReset)(nil)).Column("organization_id").Where("app_id = ? AND user_id IN (?)", appID, users)). + Union(s.db.NewSelect().Model((*LimitGrant)(nil)).Column("organization_id").Where("app_id = ? AND user_id IN (?)", appID, users)). + Scan(ctx, &shared) + if err != nil { return nil, stack.Wrap(fmt.Errorf("store: user organizations: %w", err)) } organizations := map[string]bool{} @@ -121,16 +126,15 @@ func (s *Store) ForgetUserSettings(ctx context.Context, appID, userID, replaceme err := s.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { for _, change := range []struct { count *int64 - query string - args []any + exec func(context.Context, ...any) (sql.Result, error) }{ - {&counts.Policies, "DELETE FROM policies WHERE scope = ? AND scope_id = ?", []any{ScopeUser, UserScopeID(appID, userID)}}, - {&counts.Limits, "DELETE FROM limits WHERE level = ? AND app_id = ? AND user_id = ?", []any{LevelUser, appID, userID}}, - {&counts.Resets, "UPDATE usage_resets SET user_id = ? WHERE app_id = ? AND user_id = ?", []any{replacement, appID, userID}}, - {&counts.Grants, "UPDATE limit_grants SET user_id = ? WHERE app_id = ? AND user_id = ?", []any{replacement, appID, userID}}, - {&counts.Campaigns, "UPDATE campaigns SET user_id = ? WHERE customer_id = ? AND user_id = ?", []any{replacement, appID, userID}}, + {&counts.Policies, tx.NewDelete().Model((*Policy)(nil)).Where("scope = ? AND scope_id = ?", ScopeUser, UserScopeID(appID, userID)).Exec}, + {&counts.Limits, tx.NewDelete().Model((*Limit)(nil)).Where("level = ? AND app_id = ? AND user_id = ?", LevelUser, appID, userID).Exec}, + {&counts.Resets, tx.NewUpdate().Model((*UsageReset)(nil)).Set("user_id = ?", replacement).Where("app_id = ? AND user_id = ?", appID, userID).Exec}, + {&counts.Grants, tx.NewUpdate().Model((*LimitGrant)(nil)).Set("user_id = ?", replacement).Where("app_id = ? AND user_id = ?", appID, userID).Exec}, + {&counts.Campaigns, tx.NewUpdate().Model((*Campaign)(nil)).Set("user_id = ?", replacement).Where("customer_id = ? AND user_id = ?", appID, userID).Exec}, } { - result, err := tx.ExecContext(ctx, change.query, change.args...) + result, err := change.exec(ctx) if err != nil { return err } diff --git a/acceleration/internal/store/grants.go b/acceleration/internal/store/grants.go index 572671c64..03e5ee4de 100644 --- a/acceleration/internal/store/grants.go +++ b/acceleration/internal/store/grants.go @@ -402,12 +402,17 @@ func addGrantUses(ctx context.Context, tx bun.Tx, uses []GrantUse) (int, error) for i, use := range uses { grants[i], usageIDs[i], micros[i] = use.GrantID, use.UsageID.String(), use.Micros } - if _, err := tx.ExecContext(ctx, "SELECT id FROM limit_grants WHERE id = ANY(?::bigint[]) ORDER BY id FOR UPDATE", - pgdialect.Array(slices.Compact(slices.Sorted(slices.Values(grants))))); err != nil { + _, err := tx.NewSelect().Model((*LimitGrant)(nil)). + Column("lg.id"). + Where("lg.id = ANY(?::bigint[])", pgdialect.Array(slices.Compact(slices.Sorted(slices.Values(grants))))). + Order("lg.id"). + For("UPDATE"). + Exec(ctx) + if err != nil { return 0, err } var added int - err := tx.NewRaw(` + err = tx.NewRaw(` WITH sent AS ( SELECT * FROM unnest(?::bigint[], ?::uuid[], ?::bigint[]) AS s (grant_id, usage_id, micros) ), counted AS ( diff --git a/acceleration/internal/store/policies_test.go b/acceleration/internal/store/policies_test.go index 666be8954..9071a1180 100644 --- a/acceleration/internal/store/policies_test.go +++ b/acceleration/internal/store/policies_test.go @@ -3,6 +3,7 @@ package store import ( + "slices" "time" "github.com/google/uuid" @@ -155,6 +156,28 @@ func (s *StoreSuite) TestAnAppIsInTheOrganizationItIsRegisteredOrSeenUnder() { } } +func (s *StoreSuite) TestAnOrganizationsAppsAreTheOnesSeenOrRegisteredUnderItAndItsOwn() { + org := Organization{Name: "registering"} + s.Require().NoError(s.store.CreateOrganization(s.ctx, &org)) + registered := App{OrganizationID: org.ID, Name: "registered"} + s.Require().NoError(s.store.CreateApp(s.ctx, ®istered)) + moved := App{OrganizationID: org.ID, Name: "moved"} + s.Require().NoError(s.store.CreateApp(s.ctx, &moved)) + s.Require().NoError(s.store.JoinOrganization(s.ctx, moved.ID, "org-other")) + s.Require().NoError(s.store.JoinOrganization(s.ctx, "app-seen-b", org.ID)) + s.Require().NoError(s.store.JoinOrganization(s.ctx, "app-seen-a", org.ID)) + s.Require().NoError(s.store.JoinOrganization(s.ctx, "app-elsewhere", "org-other")) + + apps, err := s.store.OrganizationApps(s.ctx, org.ID) + + s.Require().NoError(err) + s.Require().NotEmpty(apps) + s.Equal(org.ID, apps[0], "its own id first") + s.ElementsMatch([]string{org.ID, registered.ID, "app-seen-a", "app-seen-b"}, apps, + "not the app a request named another organization for, nor another organization's") + s.Less(slices.Index(apps, "app-seen-a"), slices.Index(apps, "app-seen-b"), "in order") +} + func scopeIDs(policies []Policy) []string { ids := make([]string, len(policies)) for i, policy := range policies { diff --git a/acceleration/internal/store/price_changes.go b/acceleration/internal/store/price_changes.go index b8ef9d895..42e0bd717 100644 --- a/acceleration/internal/store/price_changes.go +++ b/acceleration/internal/store/price_changes.go @@ -258,8 +258,8 @@ func (s *Store) CreatePriceChange(ctx context.Context, write PriceChangeWrite, return err } } - var seeded bool - if err := tx.QueryRowContext(ctx, "SELECT EXISTS (SELECT 1 FROM price_changes)").Scan(&seeded); err != nil { + seeded, err := tx.NewSelect().Model((*PriceChange)(nil)).Exists(ctx) + if err != nil { return err } if !seeded { @@ -289,8 +289,8 @@ func (s *Store) CreatePriceChange(ctx context.Context, write PriceChangeWrite, CreatedBy: write.ActorID, IdempotencyKey: write.IdempotencyKey, } - var taken bool - if err := tx.QueryRowContext(ctx, "SELECT EXISTS (SELECT 1 FROM price_changes WHERE name = ?)", change.Name).Scan(&taken); err != nil { + taken, err := tx.NewSelect().Model((*PriceChange)(nil)).Where("name = ?", change.Name).Exists(ctx) + if err != nil { return err } if taken { diff --git a/acceleration/internal/store/prices.go b/acceleration/internal/store/prices.go index b2d5a1279..60939a043 100644 --- a/acceleration/internal/store/prices.go +++ b/acceleration/internal/store/prices.go @@ -417,13 +417,13 @@ func (s *Store) ActivePriceEntries(ctx context.Context, at time.Time) ([]PriceEn func activePriceEntries(ctx context.Context, db bun.IDB, at time.Time) ([]PriceEntry, error) { var newest []PriceEntry - err := db.NewRaw(` -SELECT DISTINCT ON (pe.modality, pe.provider, pe.model, pe.region, pe.byok, pe.attributes) `+qualified("pe", priceEntryColumns)+` -FROM price_list_entries AS pe -JOIN price_changes AS pc ON pc.name = pe.change AND pc.cancelled_at IS NULL -WHERE pe.effective_at <= ? -ORDER BY pe.modality, pe.provider, pe.model, pe.region, pe.byok, pe.attributes, pe.effective_at DESC, pe.change DESC, pe.id DESC`, at). - Scan(ctx, &newest) + err := db.NewSelect().Model(&newest). + DistinctOn("pe.modality, pe.provider, pe.model, pe.region, pe.byok, pe.attributes"). + Join("JOIN price_changes AS pc ON pc.name = pe.change AND pc.cancelled_at IS NULL"). + Where("pe.effective_at <= ?", at). + Order("pe.modality", "pe.provider", "pe.model", "pe.region", "pe.byok", "pe.attributes"). + OrderExpr("pe.effective_at DESC, pe.change DESC, pe.id DESC"). + Scan(ctx) if err != nil { return nil, stack.Wrap(fmt.Errorf("store: active price entries: %w", err)) } @@ -436,15 +436,6 @@ ORDER BY pe.modality, pe.provider, pe.model, pe.region, pe.byok, pe.attributes, return active, nil } -// qualified is a list of columns each prefixed with the table's alias. -func qualified(alias, columns string) string { - names := strings.Split(columns, ", ") - for i, name := range names { - names[i] = alias + "." + name - } - return strings.Join(names, ", ") -} - // NextPriceChangeAt is when the next change not cancelled takes effect after after, and // whether one is scheduled, so a list in memory can switch at that moment. func (s *Store) NextPriceChangeAt(ctx context.Context, after time.Time) (time.Time, bool, error) { diff --git a/acceleration/internal/store/store.go b/acceleration/internal/store/store.go index 4a48acbfc..9870fe234 100644 --- a/acceleration/internal/store/store.go +++ b/acceleration/internal/store/store.go @@ -876,12 +876,16 @@ func spendGroups(groupBy string, acrossApps bool) (value, join string, ok bool) func (s *Store) OrganizationApps(ctx context.Context, organizationID string) ([]string, error) { // One statement, so an app a request records under the organization meanwhile is in one // half or the other. + namedOther := s.db.NewSelect().Model((*AppOrganization)(nil)). + ColumnExpr("1"). + Where("ao.app_id = ap.id AND ao.organization_id <> ?", organizationID) + named := s.db.NewSelect().Model((*AppOrganization)(nil)).Column("app_id").Where("organization_id = ?", organizationID). + Union(s.db.NewSelect().Model((*App)(nil)). + Column("id"). + Where("ap.organization_id = ?", organizationID). + Where("NOT EXISTS (?)", namedOther)) var members []string - err := s.db.NewRaw(`SELECT app_id FROM app_organizations WHERE organization_id = ? -UNION -SELECT id FROM apps AS ap WHERE organization_id = ? - AND NOT EXISTS (SELECT 1 FROM app_organizations AS ao WHERE ao.app_id = ap.id AND ao.organization_id <> ?) -ORDER BY 1`, organizationID, organizationID, organizationID).Scan(ctx, &members) + err := s.db.NewSelect().TableExpr("(?) AS member", named).Column("member.app_id").OrderExpr("1").Scan(ctx, &members) if err != nil { return nil, stack.Wrap(fmt.Errorf("store: organization apps: %w", err)) } @@ -1092,15 +1096,16 @@ func (s *Store) CustomerTagKeys( if err != nil || read.rows == 0 { return read, err } - err = db.NewRaw(` -SELECT tag.key AS key, tag.value AS value, - SUM(r.cost_micros) AS cost_micros_total, `+requestCount+` AS request_count -FROM requests AS r -CROSS JOIN LATERAL jsonb_each_text(r.tags) AS tag(key, value) -WHERE r.customer_id = ? - AND r.started_at >= ? AND r.started_at < ? - AND r.tags @> ?::jsonb -GROUP BY tag.key, tag.value`, customerID, from, to, string(filter)).Scan(ctx, &read.values) + err = db.NewSelect().Model((*Request)(nil)). + ColumnExpr("tag.key AS key, tag.value AS value"). + ColumnExpr("SUM(r.cost_micros) AS cost_micros_total"). + ColumnExpr(requestCount+" AS request_count"). + Join("CROSS JOIN LATERAL jsonb_each_text(r.tags) AS tag(key, value)"). + Where("r.customer_id = ?", customerID). + Where("r.started_at >= ? AND r.started_at < ?", from, to). + Where("r.tags @> ?::jsonb", string(filter)). + Group("tag.key", "tag.value"). + Scan(ctx, &read.values) return read, err }) if err != nil { @@ -1606,15 +1611,16 @@ func (s *Store) CustomerTurns(ctx context.Context, customerID, configID string, } counts, err := everyShard(ctx, s, func(ctx context.Context, db bun.IDB) (SessionTurns, error) { var counts SessionTurns - err := db.NewRaw(` -SELECT COUNT(DISTINCT a.session_id) AS sessions, COUNT(*) AS turns -FROM agent_responses AS a -JOIN agent_sessions AS s ON s.id = a.session_id -WHERE a.customer_id = ?0 AND (?1 = '' OR s.config_id = ?1) - AND a.created_at >= ?2 AND a.created_at < ?3 - AND a.parent_response_id IS NULL AND a.rewound_at IS NULL`, - customerID, configID, from, to, - ).Scan(ctx, &counts) + q := db.NewSelect().Model((*AgentResponse)(nil)). + ColumnExpr("COUNT(DISTINCT ars.session_id) AS sessions, COUNT(*) AS turns"). + Join("JOIN agent_sessions AS asn ON asn.id = ars.session_id"). + Where("ars.customer_id = ?", customerID). + Where("ars.created_at >= ? AND ars.created_at < ?", from, to). + Where("ars.parent_response_id IS NULL AND ars.rewound_at IS NULL") + if configID != "" { + q.Where("asn.config_id = ?", configID) + } + err := q.Scan(ctx, &counts) return counts, err }) if err != nil { diff --git a/acceleration/internal/store/user_phones.go b/acceleration/internal/store/user_phones.go index 6caf62340..e105d8f77 100644 --- a/acceleration/internal/store/user_phones.go +++ b/acceleration/internal/store/user_phones.go @@ -178,8 +178,12 @@ func (s *Store) LinkedPhone(ctx context.Context, customerID string, app int64, u Phone string `bun:"phone"` Since *time.Time `bun:"since"` } - err := s.db.NewRaw(`SELECT up.phone, `+historyBoundary+` AS since FROM user_phones AS up `+historyJoin+` -WHERE up.customer_id = ? AND up.stream_app_pk = ? AND up.user_id = ?`, customerID, s.linkApp(app), userID).Scan(ctx, &rows) + err := s.db.NewSelect().Model((*UserPhone)(nil)). + Column("up.phone"). + ColumnExpr(historyBoundary+" AS since"). + Join(historyJoin). + Where("up.customer_id = ? AND up.stream_app_pk = ? AND up.user_id = ?", customerID, s.linkApp(app), userID). + Scan(ctx, &rows) if err != nil { return "", nil, false, stack.Wrap(fmt.Errorf("store: linked phone: %w", err)) } @@ -411,15 +415,19 @@ const reachedAt = `regexp_replace(cm.address, '^` + retiredPrefix + `[^:]*:', '' // counts as on: it writes the same episodes, which a session reads once the config turns on. // In tx, under the link's lock. func (s *Store) retireHistory(ctx context.Context, tx bun.Tx, link UserPhone) error { - _, err := tx.NewRaw(`UPDATE contact_map AS cm -SET address = ?4 || cm.id || ':' || cm.address, updated_at = ?5 -WHERE cm.customer_id = ?0 - AND cm.agent_config_id IN (SELECT ac.id FROM agent_configs AS ac WHERE ac.customer_id = ?0 AND (ac.episode_cards OR ac.episode_cards_record)) - AND cm.kind = ?1 AND cm.address = ?2 - AND COALESCE(NULLIF(cm.stream_app_pk, ?3), 0) = ?6 - AND EXISTS (SELECT 1 FROM episodes AS ep WHERE ep.contact_id = cm.id)`, - link.CustomerID, ContactPhone, link.Phone, s.streamPins().deployment(), retiredPrefix, time.Now().UTC(), - link.StreamAppPK). + writesCards := tx.NewSelect().Model((*AgentConfig)(nil)). + Column("ac.id"). + Where("ac.customer_id = ?", link.CustomerID). + Where("ac.episode_cards OR ac.episode_cards_record") + withEpisode := tx.NewSelect().Model((*Episode)(nil)).ColumnExpr("1").Where("ep.contact_id = cm.id") + _, err := tx.NewUpdate().Model((*ContactMapEntry)(nil)). + Set("address = ? || cm.id || ':' || cm.address", retiredPrefix). + Set("updated_at = ?", time.Now().UTC()). + Where("cm.customer_id = ?", link.CustomerID). + Where("cm.agent_config_id IN (?)", writesCards). + Where("cm.kind = ? AND cm.address = ?", ContactPhone, link.Phone). + Where("COALESCE(NULLIF(cm.stream_app_pk, ?), 0) = ?", s.streamPins().deployment(), link.StreamAppPK). + Where("EXISTS (?)", withEpisode). Exec(ctx) if err != nil { return stack.Wrap(fmt.Errorf("store: retire a linked number's history: %w", err)) diff --git a/acceleration/internal/usage/ledger/forget.go b/acceleration/internal/usage/ledger/forget.go index 4ec298f6c..eb5161f80 100644 --- a/acceleration/internal/usage/ledger/forget.go +++ b/acceleration/internal/usage/ledger/forget.go @@ -25,17 +25,33 @@ func (l *Ledger) ForgetUser(ctx context.Context, forget usage.ForgetUser, actor var events, windows []counted err := l.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { if forget.DryRun { - if err := tx.NewRaw(`SELECT organization_id, count(*) FROM usage_events WHERE app_id = ? AND user_id = ? - GROUP BY organization_id`, forget.AppID, forget.UserID).Scan(ctx, &events); err != nil { + err := tx.NewSelect().Model((*UsageEvent)(nil)). + Column("organization_id"). + ColumnExpr("count(*) AS count"). + Where("app_id = ? AND user_id = ?", forget.AppID, forget.UserID). + Group("organization_id"). + Scan(ctx, &events) + if err != nil { return err } - return tx.NewRaw(`SELECT organization_id, count(*) FROM usage_windows WHERE subject = ? AND app_id = ? AND user_id = ? - GROUP BY organization_id`, usage.SubjectUser, forget.AppID, forget.UserID).Scan(ctx, &windows) + return tx.NewSelect().Model((*UsageWindow)(nil)). + Column("organization_id"). + ColumnExpr("count(*) AS count"). + Where("subject = ? AND app_id = ? AND user_id = ?", usage.SubjectUser, forget.AppID, forget.UserID). + Group("organization_id"). + Scan(ctx, &windows) } - if err := tx.NewRaw(`WITH moved AS ( - UPDATE usage_events SET user_id = ? WHERE app_id = ? AND user_id = ? RETURNING organization_id) - SELECT organization_id, count(*) FROM moved GROUP BY organization_id`, - forget.Replacement, forget.AppID, forget.UserID).Scan(ctx, &events); err != nil { + moved := tx.NewUpdate().Model((*UsageEvent)(nil)). + Set("user_id = ?", forget.Replacement). + Where("app_id = ? AND user_id = ?", forget.AppID, forget.UserID). + Returning("organization_id") + err := tx.NewSelect().With("moved", moved). + Table("moved"). + Column("organization_id"). + ColumnExpr("count(*) AS count"). + Group("organization_id"). + Scan(ctx, &events) + if err != nil { return err } if err := tx.NewRaw(`WITH moved AS ( diff --git a/acceleration/internal/usage/ledger/ledger.go b/acceleration/internal/usage/ledger/ledger.go index 55005fd11..e7aad4a8e 100644 --- a/acceleration/internal/usage/ledger/ledger.go +++ b/acceleration/internal/usage/ledger/ledger.go @@ -383,8 +383,8 @@ func storedWindows(ctx context.Context, db bun.IDB, wanted []UsageWindow) ([]Usa func (l *Ledger) PruneWindows(ctx context.Context, cutoff time.Time) (int64, error) { var deleted int64 for { - result, err := l.db.NewRaw(`DELETE FROM usage_windows WHERE ctid IN ( - SELECT ctid FROM usage_windows WHERE window_end < ? LIMIT ?)`, cutoff, pruneBatch).Exec(ctx) + ended := l.db.NewSelect().Model((*UsageWindow)(nil)).Column("ctid").Where("window_end < ?", cutoff).Limit(pruneBatch) + result, err := l.db.NewDelete().Model((*UsageWindow)(nil)).Where("ctid IN (?)", ended).Exec(ctx) if err != nil { return deleted, fmt.Errorf("ledger: prune windows: %w", err) } From 79f7dd341f780146530328309f6f44bab97abcdc Mon Sep 17 00:00:00 2001 From: Thierry Schellenbach Date: Thu, 19 Nov 2026 13:36:13 -0700 Subject: [PATCH 6/6] fix(store): keep the channel inbound queue insert raw bun writes a JSON \u0000 as the text \\u0000, so a delivery whose headers jsonb cannot hold was queued and claimed instead of failing, and the provider's retry found it taken. The insert goes back to json.Marshal and ?::jsonb, which TestAMessageTheStoreFailsToQueueIsNotClaimed checks. Co-authored-by: Cursor --- .../internal/store/channel_inbound.go | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/acceleration/internal/store/channel_inbound.go b/acceleration/internal/store/channel_inbound.go index d4c85ce5e..136d9163d 100644 --- a/acceleration/internal/store/channel_inbound.go +++ b/acceleration/internal/store/channel_inbound.go @@ -215,12 +215,28 @@ func queueChannelInbound(ctx context.Context, tx bun.Tx, message ChannelInbound) if _, err := tx.NewRaw("SELECT 1 FROM channel_threads WHERE channel_id = ? FOR NO KEY UPDATE", message.ChannelID).Exec(ctx); err != nil { return false, stack.Wrap(fmt.Errorf("store: queue channel inbound: %w", err)) } - message.NextAttemptAt = message.NextAttemptAt.UTC() - result, err := tx.NewInsert().Model(&message). - Column("channel_id", "provider_message_id", "customer_id", "connector_id", "provider_unit_id", "thread_key", - "config_id", "config_name", "author_id", "text", "media", "raw", "forward", "next_attempt_at"). - On("CONFLICT (channel_id, provider_message_id) DO NOTHING"). - Exec(ctx) + // Not the insert builder: bun writes a JSON \u0000 as the text \\u0000, and a delivery jsonb + // cannot hold must fail here, so it stays unclaimed and the provider's retry takes it again. + media, err := json.Marshal(message.Media) + if err != nil { + return false, stack.Wrap(err) + } + var forward *string + if message.Forward != nil { + encoded, err := json.Marshal(message.Forward) + if err != nil { + return false, stack.Wrap(err) + } + forward = new(string(encoded)) + } + result, err := tx.NewRaw(` +INSERT INTO channel_thread_inbound (channel_id, provider_message_id, customer_id, connector_id, provider_unit_id, thread_key, + config_id, config_name, author_id, text, media, raw, forward, next_attempt_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::jsonb, ?, ?::jsonb, ?) +ON CONFLICT (channel_id, provider_message_id) DO NOTHING`, + message.ChannelID, message.ProviderMessageID, message.CustomerID, message.ConnectorID, message.ProviderUnitID, message.ThreadKey, + message.ConfigID, message.ConfigName, message.AuthorID, message.Text, string(media), message.Raw, forward, + message.NextAttemptAt.UTC()).Exec(ctx) if err != nil { return false, stack.Wrap(fmt.Errorf("store: queue channel inbound: %w", err)) }