From 2afe90513c5c1fad8fb2cc61a442c367544326d9 Mon Sep 17 00:00:00 2001 From: Thierry Schellenbach Date: Thu, 19 Nov 2026 12:53:35 -0700 Subject: [PATCH 1/7] fix(store): give the tool groups migration its own version 20270102090000_agent_config_tool_groups.sql shared its version with 20270102090000_ui_instructions.sql, and goose refuses to run with a duplicate, so the router could not migrate. It is 20270103090000 now. TestARedactedCallStillReportsWhatItAndItsToolCost set redact_at on the global database, so under the sharded suite RedactToolOutput, which runs on the shard, found nothing. Co-authored-by: Cursor --- acceleration/internal/store/models.go | 2 +- acceleration/internal/store/tool_groups_migration_test.go | 6 +++--- acceleration/internal/store/usage_test.go | 2 +- ...oups.sql => 20270103090000_agent_config_tool_groups.sql} | 0 4 files changed, 5 insertions(+), 5 deletions(-) rename acceleration/migrations/{20270102090000_agent_config_tool_groups.sql => 20270103090000_agent_config_tool_groups.sql} (100%) diff --git a/acceleration/internal/store/models.go b/acceleration/internal/store/models.go index 61d8b82a8..cd08f6901 100644 --- a/acceleration/internal/store/models.go +++ b/acceleration/internal/store/models.go @@ -653,7 +653,7 @@ type AgentConfig struct { // EpisodeCards off; WritesCards is the two together. EpisodeCardsRecord bool `bun:"episode_cards_record,notnull"` // ToolGroups are the functions an SDK provides that the config's sessions may offer, and - // how each group is loaded (20270102090000_agent_config_tool_groups.sql). A function no + // how each group is loaded (20270103090000_agent_config_tool_groups.sql). A function no // group names is not offered. The row's progressive_tools and tool_output_retention // columns, which groups and connector bindings replaced, are not modelled: a router still // running the release before reads them, so they are left in place. diff --git a/acceleration/internal/store/tool_groups_migration_test.go b/acceleration/internal/store/tool_groups_migration_test.go index 2b880f02c..6cd918978 100644 --- a/acceleration/internal/store/tool_groups_migration_test.go +++ b/acceleration/internal/store/tool_groups_migration_test.go @@ -15,10 +15,10 @@ import ( "github.com/GetStream/Vision-Agents/acceleration/migrations" ) -// Versions either side of 20270102090000_agent_config_tool_groups.sql. +// Versions either side of 20270103090000_agent_config_tool_groups.sql. const ( - schemaBeforeToolGroups int64 = 20270101090000 - toolGroupsVersion int64 = 20270102090000 + schemaBeforeToolGroups int64 = 20270102090000 + toolGroupsVersion int64 = 20270103090000 ) // ToolGroupsMigrationSuite stops a database of its own short of tool groups, writes configs diff --git a/acceleration/internal/store/usage_test.go b/acceleration/internal/store/usage_test.go index 3d1ae0914..d58873a0f 100644 --- a/acceleration/internal/store/usage_test.go +++ b/acceleration/internal/store/usage_test.go @@ -530,7 +530,7 @@ func (s *StoreSuite) TestARedactedCallStillReportsWhatItAndItsToolCost() { s.calledTool(session, "one-1", "lookup_order", s.base, map[string]any{"call_id": "c1", "loading": ToolLoadingSummary, "definition_tokens": 461, "summary_tokens": 84}, map[string]any{"call_id": "c1", "arguments": `{"order":"A-1"}`, "tokens": 250}) - _, err := s.store.db.NewUpdate().Model((*AgentResponseItem)(nil)). + _, err := s.store.ShardDB().NewUpdate().Model((*AgentResponseItem)(nil)). Set("redact_at = ?", s.base).Where("session_id = ?", "one").Exec(s.ctx) s.Require().NoError(err) redacted, err := s.store.RedactToolOutput(s.ctx, s.base.Add(time.Hour), 100) diff --git a/acceleration/migrations/20270102090000_agent_config_tool_groups.sql b/acceleration/migrations/20270103090000_agent_config_tool_groups.sql similarity index 100% rename from acceleration/migrations/20270102090000_agent_config_tool_groups.sql rename to acceleration/migrations/20270103090000_agent_config_tool_groups.sql From f2005b06d9390f0fe696d626b8df5cfb3a756c7b Mon Sep 17 00:00:00 2001 From: Thierry Schellenbach Date: Thu, 19 Nov 2026 12:56:21 -0700 Subject: [PATCH 2/7] refactor(store): build connector token, attempt and client queries with bun Config tokens, authorization attempts, OAuth clients, the tool re-approval read and the connector delete loop go through bun's builders. The attempt updates share openAttempt's conditions instead of repeating them. Co-authored-by: Cursor --- acceleration/internal/store/config_tokens.go | 38 +++++++------- acceleration/internal/store/connections.go | 52 ++++++++++---------- acceleration/internal/store/connectors.go | 7 +-- acceleration/internal/store/oauth_clients.go | 34 ++++++------- acceleration/internal/store/tool_pins.go | 8 +-- 5 files changed, 71 insertions(+), 68 deletions(-) diff --git a/acceleration/internal/store/config_tokens.go b/acceleration/internal/store/config_tokens.go index 80c0f16e8..974c39da5 100644 --- a/acceleration/internal/store/config_tokens.go +++ b/acceleration/internal/store/config_tokens.go @@ -7,6 +7,8 @@ import ( "fmt" "time" + "github.com/uptrace/bun" + "github.com/GetStream/Vision-Agents/acceleration/internal/stack" ) @@ -23,6 +25,8 @@ var ErrNoConnectorConfigToken = errors.New("store: no configuration token for th // router for one connector, with its refresh token, sealed by the caller as one blob. It is // what creates, updates and deletes the customer's managed provider app. type ConnectorConfigToken struct { + bun.BaseModel `bun:"table:connector_config_tokens,alias:cct"` + CustomerID string `bun:"customer_id,pk"` ConnectorID string `bun:"connector_id,pk"` TokensSealed []byte `bun:"tokens_sealed,notnull"` @@ -62,21 +66,20 @@ func (s *Store) PutConnectorConfigToken(ctx context.Context, token *ConnectorCon ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), credentialDetachedTimeout) defer cancel() now := time.Now().UTC().Truncate(time.Microsecond) - err := s.db.NewRaw(` -INSERT INTO connector_config_tokens AS cct - (customer_id, connector_id, tokens_sealed, kek_version, expires_at, created_at, updated_at) -VALUES (?, ?, ?, ?, ?, ?, ?) -ON CONFLICT (customer_id, connector_id) DO UPDATE -SET tokens_sealed = EXCLUDED.tokens_sealed, - kek_version = EXCLUDED.kek_version, - expires_at = EXCLUDED.expires_at, - updated_at = EXCLUDED.updated_at -RETURNING cct.created_at`, - token.CustomerID, token.ConnectorID, token.TokensSealed, token.KEKVersion, token.ExpiresAt.UTC(), now, now). - Scan(ctx, &token.CreatedAt) + row := *token + row.CreatedAt, row.UpdatedAt = now, now + err := s.db.NewInsert().Model(&row). + On("CONFLICT (customer_id, connector_id) DO UPDATE"). + Set("tokens_sealed = EXCLUDED.tokens_sealed"). + Set("kek_version = EXCLUDED.kek_version"). + Set("expires_at = EXCLUDED.expires_at"). + Set("updated_at = EXCLUDED.updated_at"). + Returning("cct.created_at"). + Scan(ctx) if err != nil { return stack.Wrap(fmt.Errorf("store: put connector config token: %w", err)) } + token.CreatedAt = row.CreatedAt token.UpdatedAt = now return nil } @@ -87,10 +90,9 @@ func (s *Store) ConnectorConfigToken(ctx context.Context, customerID, connectorI return ConnectorConfigToken{}, stack.Wrap(errors.New("store: a customer and a connector id are required")) } var token ConnectorConfigToken - err := s.db.NewRaw(` -SELECT customer_id, connector_id, tokens_sealed, kek_version, expires_at, created_at, updated_at -FROM connector_config_tokens WHERE customer_id = ? AND connector_id = ?`, customerID, connectorID). - Scan(ctx, &token.CustomerID, &token.ConnectorID, &token.TokensSealed, &token.KEKVersion, &token.ExpiresAt, &token.CreatedAt, &token.UpdatedAt) + err := s.db.NewSelect().Model(&token). + Where("cct.customer_id = ? AND cct.connector_id = ?", customerID, connectorID). + Scan(ctx) if errors.Is(err, sql.ErrNoRows) { return ConnectorConfigToken{}, stack.Wrap(fmt.Errorf("%w: %s", ErrNoConnectorConfigToken, connectorID)) } @@ -106,7 +108,9 @@ func (s *Store) DeleteConnectorConfigToken(ctx context.Context, customerID, conn if customerID == "" || connectorID == "" { return stack.Wrap(errors.New("store: a customer and a connector id are required")) } - _, err := s.db.ExecContext(ctx, "DELETE FROM connector_config_tokens WHERE customer_id = ? AND connector_id = ?", customerID, connectorID) + _, err := s.db.NewDelete().Model((*ConnectorConfigToken)(nil)). + Where("customer_id = ? AND connector_id = ?", customerID, connectorID). + Exec(ctx) if err != nil { return stack.Wrap(fmt.Errorf("store: delete connector config token: %w", err)) } diff --git a/acceleration/internal/store/connections.go b/acceleration/internal/store/connections.go index 5ad6418fb..79f522185 100644 --- a/acceleration/internal/store/connections.go +++ b/acceleration/internal/store/connections.go @@ -727,10 +727,10 @@ func (s *Store) ConnectorConnectionReferenced(ctx context.Context, customerID, i return false, stack.Wrap(errors.New("store: a customer and a connection id are required")) } var referenced bool - err := s.db.QueryRowContext(ctx, ` -SELECT `+boundByConfig+` -FROM connector_connections AS cc -WHERE cc.customer_id = ? AND cc.id = ? AND cc.deleted_at IS NULL`, customerID, id).Scan(&referenced) + err := s.db.NewSelect().Model((*ConnectorConnection)(nil)). + ColumnExpr(boundByConfig). + Where("cc.customer_id = ? AND cc.id = ? AND cc.deleted_at IS NULL", customerID, id). + Scan(ctx, &referenced) if errors.Is(err, sql.ErrNoRows) { return false, stack.Wrap(fmt.Errorf("%w: %s", ErrNoConnectorConnection, id)) } @@ -806,15 +806,12 @@ func (s *Store) ConsumeConnectorAuthorizationAttempt(ctx context.Context, state return ConnectorAuthorizationAttempt{}, errors.New("store: an oauth state is required") } var attempt ConnectorAuthorizationAttempt - err := s.db.NewRaw(` -UPDATE connector_authorization_attempts AS caa -SET consumed_at = now() -WHERE caa.state_hash = ? - AND caa.consumed_at IS NULL - AND caa.expires_at > now() - AND EXISTS (SELECT 1 FROM connector_connections AS cc - WHERE cc.id = caa.connection_id AND cc.customer_id = caa.customer_id AND cc.deleted_at IS NULL) -RETURNING caa.*`, AuthorizationStateHash(state)).Scan(ctx, &attempt) + err := s.db.NewUpdate().Model(&attempt). + Set("consumed_at = now()"). + Where("caa.state_hash = ?", AuthorizationStateHash(state)). + ApplyQueryBuilder(attemptOpen). + Returning("caa.*"). + Scan(ctx) if errors.Is(err, sql.ErrNoRows) { return ConnectorAuthorizationAttempt{}, ErrNoAuthorizationAttempt } @@ -833,16 +830,13 @@ func (s *Store) HandOffConnectorAuthorizationAttempt(ctx context.Context, id str if id == "" || len(previous) == 0 || len(sealed) == 0 || kekVersion < 1 { return errors.New("store: a handoff needs an attempt id, the blob it read and a new one under a key version of 1 or more") } - result, err := s.db.NewRaw(` -UPDATE connector_authorization_attempts AS caa -SET attempt_sealed = ?, kek_version = ? -WHERE caa.id = ? - AND caa.attempt_sealed = ? - AND caa.consumed_at IS NULL - AND caa.expires_at > now() - AND EXISTS (SELECT 1 FROM connector_connections AS cc - WHERE cc.id = caa.connection_id AND cc.customer_id = caa.customer_id AND cc.deleted_at IS NULL)`, - sealed, kekVersion, id, previous).Exec(ctx) + result, err := s.db.NewUpdate().Model((*ConnectorAuthorizationAttempt)(nil)). + Set("attempt_sealed = ?", sealed). + Set("kek_version = ?", kekVersion). + Where("caa.id = ?", id). + Where("caa.attempt_sealed = ?", previous). + ApplyQueryBuilder(attemptOpen). + Exec(ctx) if err != nil { return fmt.Errorf("store: hand off authorization attempt: %w", err) } @@ -863,15 +857,21 @@ func AuthorizationStateHash(state string) string { return hex.EncodeToString(hash[:]) } +// attemptOpen keeps the attempts that are unconsumed, unexpired and for a live connection. +func attemptOpen(q bun.QueryBuilder) bun.QueryBuilder { + return q. + Where("caa.consumed_at IS NULL"). + Where("caa.expires_at > now()"). + Where("EXISTS (SELECT 1 FROM connector_connections AS cc WHERE cc.id = caa.connection_id AND cc.customer_id = caa.customer_id AND cc.deleted_at IS NULL)") +} + // openAttempt is the one attempt matching where that is unconsumed, unexpired and for a // live connection. func (s *Store) openAttempt(ctx context.Context, where string, arg any) (ConnectorAuthorizationAttempt, error) { var attempt ConnectorAuthorizationAttempt err := s.db.NewSelect().Model(&attempt). Where(where, arg). - Where("caa.consumed_at IS NULL"). - Where("caa.expires_at > now()"). - Where("EXISTS (SELECT 1 FROM connector_connections AS cc WHERE cc.id = caa.connection_id AND cc.customer_id = caa.customer_id AND cc.deleted_at IS NULL)"). + ApplyQueryBuilder(attemptOpen). Scan(ctx) if errors.Is(err, sql.ErrNoRows) { return ConnectorAuthorizationAttempt{}, ErrNoAuthorizationAttempt diff --git a/acceleration/internal/store/connectors.go b/acceleration/internal/store/connectors.go index 91ad1d8bd..68371eca1 100644 --- a/acceleration/internal/store/connectors.go +++ b/acceleration/internal/store/connectors.go @@ -637,9 +637,10 @@ func (s *Store) DeleteConnectorDefinition(ctx context.Context, customerID, id st } // The deliveries go with their destination (ON DELETE CASCADE, // 20261007040000_connector_event_destinations.sql). - for _, table := range []string{"connector_oauth_clients", "connector_config_tokens", "connector_event_destinations"} { - if _, err := tx.ExecContext(ctx, "DELETE FROM "+table+" WHERE customer_id = ? AND connector_id = ?", customerID, id); err != nil { - return fmt.Errorf("store: delete the connector's %s: %w", table, err) + for _, model := range []any{(*ConnectorOAuthClient)(nil), (*ConnectorConfigToken)(nil), (*EventDestination)(nil)} { + q := tx.NewDelete().Model(model).Where("customer_id = ? AND connector_id = ?", customerID, id) + if _, err := q.Exec(ctx); err != nil { + return fmt.Errorf("store: delete the connector's %s: %w", q.GetTableName(), err) } } _, err = tx.NewDelete().Model((*ConnectorDefinition)(nil)). diff --git a/acceleration/internal/store/oauth_clients.go b/acceleration/internal/store/oauth_clients.go index f9fa226f7..739bc82ca 100644 --- a/acceleration/internal/store/oauth_clients.go +++ b/acceleration/internal/store/oauth_clients.go @@ -114,25 +114,21 @@ func (s *Store) PutConnectorOAuthClient(ctx context.Context, client *ConnectorOA return fmt.Errorf("%w: %s lists %v, not %s", ErrOAuthClientRegistrationNotListed, client.ConnectorID, definition.Manifest.Client.Registration, client.Registration) } - err = tx.NewRaw(` -INSERT INTO connector_oauth_clients AS coc - (customer_id, connector_id, registration, client_id, auth_method, secret_sealed, kek_version, - provider_app_id, stream_app_pk, signing_secret_sealed, signing_kek_version, created_at, updated_at) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) -ON CONFLICT (customer_id, connector_id) DO UPDATE -SET client_id = EXCLUDED.client_id, - auth_method = EXCLUDED.auth_method, - secret_sealed = EXCLUDED.secret_sealed, - kek_version = EXCLUDED.kek_version, - provider_app_id = EXCLUDED.provider_app_id, - signing_secret_sealed = EXCLUDED.signing_secret_sealed, - signing_kek_version = EXCLUDED.signing_kek_version, - updated_at = EXCLUDED.updated_at -WHERE coc.registration = EXCLUDED.registration -RETURNING coc.created_at, coc.stream_app_pk`, - client.CustomerID, client.ConnectorID, client.Registration, client.ClientID, client.AuthMethod, - client.SecretSealed, client.KEKVersion, client.ProviderAppID, nullablePin(client.StreamAppPK), - client.SigningSecretSealed, client.SigningKEKVersion, now, now).Scan(ctx, &createdAt, &pin) + row := *client + row.CreatedAt, row.UpdatedAt = now, now + err = tx.NewInsert().Model(&row). + On("CONFLICT (customer_id, connector_id) DO UPDATE"). + Set("client_id = EXCLUDED.client_id"). + Set("auth_method = EXCLUDED.auth_method"). + Set("secret_sealed = EXCLUDED.secret_sealed"). + Set("kek_version = EXCLUDED.kek_version"). + Set("provider_app_id = EXCLUDED.provider_app_id"). + Set("signing_secret_sealed = EXCLUDED.signing_secret_sealed"). + Set("signing_kek_version = EXCLUDED.signing_kek_version"). + Set("updated_at = EXCLUDED.updated_at"). + Where("coc.registration = EXCLUDED.registration"). + Returning("coc.created_at, coc.stream_app_pk"). + Scan(ctx, &createdAt, &pin) // The conflict's WHERE kept the row, so nothing was written or returned. if errors.Is(err, sql.ErrNoRows) { return fmt.Errorf("%w: %s", ErrOAuthClientRegistration, client.ConnectorID) diff --git a/acceleration/internal/store/tool_pins.go b/acceleration/internal/store/tool_pins.go index 02cced618..47ce48161 100644 --- a/acceleration/internal/store/tool_pins.go +++ b/acceleration/internal/store/tool_pins.go @@ -124,9 +124,11 @@ func (s *Store) ReapproveConnectorTools(ctx context.Context, connection Connecto // FOR SHARE: a consent or a credentials write that begins a new grant updates this row, // so one that comes later waits for this transaction, and one that came first is seen. var sameGrant bool - err := tx.QueryRowContext(ctx, "SELECT connected_at IS NOT DISTINCT FROM ? FROM connector_connections "+ - "WHERE customer_id = ? AND id = ? AND deleted_at IS NULL FOR SHARE", - connection.ConnectedAt, connection.CustomerID, connection.ID).Scan(&sameGrant) + err := tx.NewSelect().Model((*ConnectorConnection)(nil)). + ColumnExpr("cc.connected_at IS NOT DISTINCT FROM ?", connection.ConnectedAt). + Where("cc.customer_id = ? AND cc.id = ? AND cc.deleted_at IS NULL", connection.CustomerID, connection.ID). + For("SHARE"). + Scan(ctx, &sameGrant) if errors.Is(err, sql.ErrNoRows) { return fmt.Errorf("%w: %s", ErrNoConnectorConnection, connection.ID) } From 5c49d86f7ec61b907121ca44c29e9755fcaeae7a Mon Sep 17 00:00:00 2001 From: Thierry Schellenbach Date: Thu, 19 Nov 2026 13:06:19 -0700 Subject: [PATCH 3/7] refactor(store): build inbound queue and event delivery queries with bun The chat and channel inbound queues (all but their claims and the thread lock, whose SQL text tests match), balance notice deliveries and event destination rotation go through bun's builders. The models' jsonb types marshal themselves, so the hand-written json.Marshal and ?::jsonb casts go. Co-authored-by: Cursor --- .../internal/store/balance_notices.go | 19 +++-- .../internal/store/channel_inbound.go | 70 ++++++++----------- acceleration/internal/store/chat_inbound.go | 58 ++++++++------- .../internal/store/event_destinations.go | 22 +++--- 4 files changed, 84 insertions(+), 85 deletions(-) diff --git a/acceleration/internal/store/balance_notices.go b/acceleration/internal/store/balance_notices.go index 77e0a8c26..f61c09dbd 100644 --- a/acceleration/internal/store/balance_notices.go +++ b/acceleration/internal/store/balance_notices.go @@ -2,7 +2,6 @@ package store import ( "context" - "encoding/json" "errors" "fmt" "time" @@ -49,10 +48,6 @@ func (s *Store) NoteBalance(ctx context.Context, notice BalanceNotice, delivery if notice.Kind != NoticeBalanceLow && notice.Kind != NoticeBalanceExhausted { return false, 0, stack.Wrap(fmt.Errorf("store: a balance notice is %s or %s, not %q", NoticeBalanceLow, NoticeBalanceExhausted, notice.Kind)) } - headers, err := json.Marshal(delivery.Headers) - if err != nil { - return false, 0, stack.Wrap(err) - } err = s.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { written, err := tx.NewInsert().Model(¬ice).On("CONFLICT DO NOTHING").Exec(ctx) if err != nil { @@ -80,11 +75,15 @@ func (s *Store) NoteBalance(ctx context.Context, notice BalanceNotice, delivery if err != nil { return err } - if _, err := tx.NewRaw(` -INSERT INTO connector_event_deliveries (destination_id, id, headers, body, attempts, next_attempt_at) -VALUES (?, ?, ?::jsonb, ?, 0, ?) -ON CONFLICT (destination_id, id) DO NOTHING`, - destination.ID, delivery.ID, string(headers), payload, delivery.NextAttemptAt.UTC()).Exec(ctx); err != nil { + row := EventDelivery{ + DestinationID: destination.ID, ID: delivery.ID, Headers: delivery.Headers, Body: payload, + NextAttemptAt: delivery.NextAttemptAt.UTC(), + } + _, err = tx.NewInsert().Model(&row). + Column("destination_id", "id", "headers", "body", "attempts", "next_attempt_at"). + On("CONFLICT (destination_id, id) DO NOTHING"). + Exec(ctx) + if err != nil { return err } queued++ diff --git a/acceleration/internal/store/channel_inbound.go b/acceleration/internal/store/channel_inbound.go index ac9973789..d4c85ce5e 100644 --- a/acceleration/internal/store/channel_inbound.go +++ b/acceleration/internal/store/channel_inbound.go @@ -215,26 +215,12 @@ func queueChannelInbound(ctx context.Context, tx bun.Tx, message ChannelInbound) if _, err := tx.NewRaw("SELECT 1 FROM channel_threads WHERE channel_id = ? FOR NO KEY UPDATE", message.ChannelID).Exec(ctx); err != nil { return false, stack.Wrap(fmt.Errorf("store: queue channel inbound: %w", err)) } - media, err := json.Marshal(message.Media) - if err != nil { - return false, stack.Wrap(err) - } - var forward *string - if message.Forward != nil { - encoded, err := json.Marshal(message.Forward) - if err != nil { - return false, stack.Wrap(err) - } - forward = new(string(encoded)) - } - result, err := tx.NewRaw(` -INSERT INTO channel_thread_inbound (channel_id, provider_message_id, customer_id, connector_id, provider_unit_id, thread_key, - config_id, config_name, author_id, text, media, raw, forward, next_attempt_at) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::jsonb, ?, ?::jsonb, ?) -ON CONFLICT (channel_id, provider_message_id) DO NOTHING`, - message.ChannelID, message.ProviderMessageID, message.CustomerID, message.ConnectorID, message.ProviderUnitID, message.ThreadKey, - message.ConfigID, message.ConfigName, message.AuthorID, message.Text, string(media), message.Raw, forward, - message.NextAttemptAt.UTC()).Exec(ctx) + message.NextAttemptAt = message.NextAttemptAt.UTC() + result, err := tx.NewInsert().Model(&message). + Column("channel_id", "provider_message_id", "customer_id", "connector_id", "provider_unit_id", "thread_key", + "config_id", "config_name", "author_id", "text", "media", "raw", "forward", "next_attempt_at"). + On("CONFLICT (channel_id, provider_message_id) DO NOTHING"). + Exec(ctx) if err != nil { return false, stack.Wrap(fmt.Errorf("store: queue channel inbound: %w", err)) } @@ -309,13 +295,13 @@ func (s *Store) claimChannelInbound(ctx context.Context, query string, now time. // found is false when no message is queued at all. func (s *Store) NextChannelInboundAt(ctx context.Context) (next time.Time, found bool, err error) { var at sql.NullTime - if err := s.db.NewRaw(` -SELECT min(head.next_attempt_at) FROM ( - SELECT DISTINCT ON (channel_id) next_attempt_at - FROM channel_thread_inbound - WHERE state = 'queued' - ORDER BY channel_id, seq -) AS head`).Scan(ctx, &at); err != nil { + head := s.db.NewSelect().Model((*ChannelInbound)(nil)). + DistinctOn("cti.channel_id"). + Column("cti.next_attempt_at"). + Where("cti.state = ?", ChannelInboundQueued). + Order("cti.channel_id", "cti.seq") + err = s.db.NewSelect().TableExpr("(?) AS head", head).ColumnExpr("min(head.next_attempt_at)").Scan(ctx, &at) + if err != nil { return time.Time{}, false, stack.Wrap(fmt.Errorf("store: next channel inbound: %w", err)) } return at.Time, at.Valid, nil @@ -357,8 +343,11 @@ func (s *Store) FailChannelInbound(ctx context.Context, claimed ChannelInbound, if forward(ctx) != nil { return nil } - if _, err := tx.NewRaw("UPDATE channel_thread_inbound SET forward = jsonb_set(forward, '{forwarded}', 'true') WHERE seq = ?", - claimed.Seq).Exec(ctx); err != nil { + _, err = tx.NewUpdate().Model((*ChannelInbound)(nil)). + Set("forward = jsonb_set(forward, '{forwarded}', 'true')"). + Where("seq = ?", claimed.Seq). + Exec(ctx) + if err != nil { return fmt.Errorf("store: fail channel inbound: %w", err) } return nil @@ -383,13 +372,14 @@ func lockChannelInboundDelivery(ctx context.Context, tx bun.Tx, claimed ChannelI // it: one with the same provider id for the delivery, or with none and the same body, as the // event destinations key a forward (eventforward.deliveryID). func channelInboundDeliveryForwarded(ctx context.Context, tx bun.Tx, claimed ChannelInbound) (bool, error) { - var forwarded bool - if err := tx.NewRaw(` -SELECT EXISTS ( - SELECT 1 FROM channel_thread_inbound - WHERE state = 'failed' AND forward->>'forwarded' = 'true' AND customer_id = ? AND connector_id = ? - AND forward->>'id' = ? AND (? <> '' OR raw = ?) -)`, claimed.CustomerID, claimed.ConnectorID, claimed.Forward.ID, claimed.Forward.ID, claimed.Raw).Scan(ctx, &forwarded); err != nil { + forwarded, err := tx.NewSelect().Model((*ChannelInbound)(nil)). + Where("state = ?", ChannelInboundFailed). + Where("forward->>'forwarded' = 'true'"). + Where("customer_id = ? AND connector_id = ?", claimed.CustomerID, claimed.ConnectorID). + Where("forward->>'id' = ?", claimed.Forward.ID). + Where("(? <> '' OR raw = ?)", claimed.Forward.ID, claimed.Raw). + Exists(ctx) + if err != nil { return false, fmt.Errorf("store: fail channel inbound: %w", err) } return forwarded, nil @@ -452,10 +442,10 @@ func updateClaimedChannelInbound(ctx context.Context, claimed ChannelInbound, up // PruneChannelInbound deletes the rows of written messages finished a day before now, and of // given-up ones 30 days before, and returns how many it deleted. func (s *Store) PruneChannelInbound(ctx context.Context, now time.Time) (int, error) { - result, err := s.db.NewRaw(` -DELETE FROM channel_thread_inbound -WHERE (state = 'done' AND finished_at < ?) OR (state = 'failed' AND finished_at < ?)`, - now.Add(-channelInboundDoneKeep).UTC(), now.Add(-channelInboundFailedKeep).UTC()).Exec(ctx) + result, err := s.db.NewDelete().Model((*ChannelInbound)(nil)). + Where("(state = ? AND finished_at < ?) OR (state = ? AND finished_at < ?)", + ChannelInboundDone, now.Add(-channelInboundDoneKeep).UTC(), ChannelInboundFailed, now.Add(-channelInboundFailedKeep).UTC()). + Exec(ctx) if err != nil { return 0, stack.Wrap(fmt.Errorf("store: prune channel inbound: %w", err)) } diff --git a/acceleration/internal/store/chat_inbound.go b/acceleration/internal/store/chat_inbound.go index 36e1ab824..142e264bc 100644 --- a/acceleration/internal/store/chat_inbound.go +++ b/acceleration/internal/store/chat_inbound.go @@ -82,12 +82,11 @@ func queueChatInbound(ctx context.Context, db bun.IDB, delivery ChatInbound) (bo if delivery.StreamApp == 0 { return false, stack.Wrap(errors.New("store: a Stream app is required: a message queued without one would be queued again once the app is known")) } - result, err := db.NewRaw(` -INSERT INTO chat_inbound (stream_app, message_id, customer_id, deployment, api_key, body, next_attempt_at) -VALUES (?, ?, ?, ?, ?, ?, ?) -ON CONFLICT (stream_app, message_id) DO NOTHING`, - delivery.StreamApp, delivery.MessageID, delivery.CustomerID, delivery.Deployment, delivery.APIKey, delivery.Body, - delivery.NextAttemptAt.UTC()).Exec(ctx) + delivery.NextAttemptAt = delivery.NextAttemptAt.UTC() + result, err := db.NewInsert().Model(&delivery). + Column("stream_app", "message_id", "customer_id", "deployment", "api_key", "body", "next_attempt_at"). + On("CONFLICT (stream_app, message_id) DO NOTHING"). + Exec(ctx) if err != nil { return false, stack.Wrap(fmt.Errorf("store: queue chat inbound: %w", err)) } @@ -164,7 +163,7 @@ func (s *Store) claimChatInbound(ctx context.Context, query string, now time.Tim // ChatInboundBySeq returns the message queued as seq, in whatever state it is. func (s *Store) ChatInboundBySeq(ctx context.Context, seq int64) (ChatInbound, error) { var delivery ChatInbound - err := s.db.NewRaw("SELECT "+chatInboundColumns+" FROM chat_inbound AS cd WHERE cd.seq = ?", seq).Scan(ctx, &delivery) + err := s.db.NewSelect().Model(&delivery).Where("cd.seq = ?", seq).Scan(ctx) if errors.Is(err, sql.ErrNoRows) { return ChatInbound{}, stack.Wrap(fmt.Errorf("%w: %d", ErrNoChatInbound, seq)) } @@ -178,7 +177,11 @@ func (s *Store) ChatInboundBySeq(ctx context.Context, seq int64) (ChatInbound, e // their lease. found is false when no message is queued at all. func (s *Store) NextChatInboundAt(ctx context.Context) (next time.Time, found bool, err error) { var at sql.NullTime - if err := s.db.NewRaw("SELECT min(next_attempt_at) FROM chat_inbound WHERE state = 'queued'").Scan(ctx, &at); err != nil { + err = s.db.NewSelect().Model((*ChatInbound)(nil)). + ColumnExpr("min(next_attempt_at)"). + Where("state = ?", ChatInboundQueued). + Scan(ctx, &at) + if err != nil { return time.Time{}, false, stack.Wrap(fmt.Errorf("store: next chat inbound: %w", err)) } return at.Time, at.Valid, nil @@ -239,10 +242,12 @@ func updateClaimedChatInbound(ctx context.Context, claimed ChatInbound, states [ // nothing is queued under streamApp and messageID: there is nothing to answer then. The lease // is left as it is, for the worker holding it to finish the row (FinishChatInbound). func (s *Store) StartChatInboundAnswer(ctx context.Context, streamApp int64, messageID string, at time.Time) (started bool, err error) { - result, err := s.db.NewRaw(` -UPDATE chat_inbound SET state = 'done', finished_at = ? -WHERE stream_app = ? AND message_id = ? AND state = 'queued'`, - at.UTC(), streamApp, messageID).Exec(ctx) + result, err := s.db.NewUpdate().Model((*ChatInbound)(nil)). + Set("state = ?", ChatInboundDone). + Set("finished_at = ?", at.UTC()). + Where("stream_app = ? AND message_id = ?", streamApp, messageID). + Where("state = ?", ChatInboundQueued). + Exec(ctx) if err != nil { return false, stack.Wrap(fmt.Errorf("store: start chat inbound answer: %w", err)) } @@ -257,10 +262,12 @@ WHERE stream_app = ? AND message_id = ? AND state = 'queued'`, // messageID said, so a later try of it hands that on rather than hearing it again. What was // kept first stays. func (s *Store) KeepChatInboundHeard(ctx context.Context, streamApp int64, messageID, heard string) error { - if _, err := s.db.NewRaw(` -UPDATE chat_inbound SET heard = ? -WHERE stream_app = ? AND message_id = ? AND heard IS NULL`, - heard, streamApp, messageID).Exec(ctx); err != nil { + _, err := s.db.NewUpdate().Model((*ChatInbound)(nil)). + Set("heard = ?", heard). + Where("stream_app = ? AND message_id = ?", streamApp, messageID). + Where("heard IS NULL"). + Exec(ctx) + if err != nil { return stack.Wrap(fmt.Errorf("store: keep chat inbound heard: %w", err)) } return nil @@ -270,10 +277,13 @@ WHERE stream_app = ? AND message_id = ? AND heard IS NULL`, // and that nothing then took, such as a dispatch worker that was not connected, so a later try // answers it. The lease is left as it is, for the worker holding it to retry the row. func (s *Store) ReleaseChatInboundAnswer(ctx context.Context, streamApp int64, messageID string) error { - if _, err := s.db.NewRaw(` -UPDATE chat_inbound SET state = 'queued', finished_at = NULL -WHERE stream_app = ? AND message_id = ? AND state = 'done'`, - streamApp, messageID).Exec(ctx); err != nil { + _, err := s.db.NewUpdate().Model((*ChatInbound)(nil)). + Set("state = ?", ChatInboundQueued). + Set("finished_at = NULL"). + Where("stream_app = ? AND message_id = ?", streamApp, messageID). + Where("state = ?", ChatInboundDone). + Exec(ctx) + if err != nil { return stack.Wrap(fmt.Errorf("store: release chat inbound answer: %w", err)) } return nil @@ -282,10 +292,10 @@ WHERE stream_app = ? AND message_id = ? AND state = 'done'`, // PruneChatInbound deletes the rows of answered messages finished a day before now, and of // given-up ones 30 days before, and returns how many it deleted. func (s *Store) PruneChatInbound(ctx context.Context, now time.Time) (int, error) { - result, err := s.db.NewRaw(` -DELETE FROM chat_inbound -WHERE (state = 'done' AND finished_at < ?) OR (state = 'failed' AND finished_at < ?)`, - now.Add(-chatInboundDoneKeep).UTC(), now.Add(-chatInboundFailedKeep).UTC()).Exec(ctx) + result, err := s.db.NewDelete().Model((*ChatInbound)(nil)). + Where("(state = ? AND finished_at < ?) OR (state = ? AND finished_at < ?)", + ChatInboundDone, now.Add(-chatInboundDoneKeep).UTC(), ChatInboundFailed, now.Add(-chatInboundFailedKeep).UTC()). + Exec(ctx) if err != nil { return 0, stack.Wrap(fmt.Errorf("store: prune chat inbound: %w", err)) } diff --git a/acceleration/internal/store/event_destinations.go b/acceleration/internal/store/event_destinations.go index af2c6c4e3..a32f5dea7 100644 --- a/acceleration/internal/store/event_destinations.go +++ b/acceleration/internal/store/event_destinations.go @@ -162,16 +162,16 @@ func (s *Store) RotateEventDestinationSecret(ctx context.Context, customerID, co } now := time.Now().UTC().Truncate(time.Microsecond) var destination EventDestination - err := s.db.NewRaw(` -UPDATE connector_event_destinations -SET previous_secret_sealed = secret_sealed, - previous_kek_version = kek_version, - previous_until = ?, - secret_sealed = ?, - kek_version = ?, - updated_at = ? -WHERE id = ? AND customer_id = ? AND connector_id = ? -RETURNING *`, previousUntil.UTC(), sealed, version, now, id, customerID, connectorID).Scan(ctx, &destination) + err := s.db.NewUpdate().Model(&destination). + Set("previous_secret_sealed = secret_sealed"). + Set("previous_kek_version = kek_version"). + Set("previous_until = ?", previousUntil.UTC()). + Set("secret_sealed = ?", sealed). + Set("kek_version = ?", version). + Set("updated_at = ?", now). + Where("id = ? AND customer_id = ? AND connector_id = ?", id, customerID, connectorID). + Returning("*"). + Scan(ctx) if errors.Is(err, sql.ErrNoRows) { return EventDestination{}, stack.Wrap(fmt.Errorf("%w: %s", ErrNoEventDestination, id)) } @@ -316,7 +316,7 @@ func (s *Store) claimEventDeliveries(ctx context.Context, query string, now time // end of their lease. found is false when no delivery is queued at all. func (s *Store) NextEventDeliveryAt(ctx context.Context) (next time.Time, found bool, err error) { var at sql.NullTime - if err := s.db.NewRaw("SELECT min(next_attempt_at) FROM connector_event_deliveries").Scan(ctx, &at); err != nil { + if err := s.db.NewSelect().Model((*EventDelivery)(nil)).ColumnExpr("min(next_attempt_at)").Scan(ctx, &at); err != nil { return time.Time{}, false, stack.Wrap(fmt.Errorf("store: next event delivery: %w", err)) } return at.Time, at.Valid, nil From dc8530d9ba6514a2ab1ebe062bab9e2a79e3e780 Mon Sep 17 00:00:00 2001 From: Thierry Schellenbach Date: Thu, 19 Nov 2026 13:08:07 -0700 Subject: [PATCH 4/7] refactor(store): build image job, reship and campaign lease claims with bun ClaimImageJobs, Reship, Unshipped, TakeOverExpired and PutOffClassification go through bun's With, TableExpr and For. Reship and Unshipped had no test; they have now, and the tests pass on the raw SQL they replace too. Co-authored-by: Cursor --- .../internal/store/campaign_dispatch.go | 36 ++++++---- acceleration/internal/store/image_jobs.go | 27 +++---- acceleration/internal/store/shipping.go | 39 +++++++---- acceleration/internal/store/shipping_test.go | 70 +++++++++++++++++++ 4 files changed, 129 insertions(+), 43 deletions(-) diff --git a/acceleration/internal/store/campaign_dispatch.go b/acceleration/internal/store/campaign_dispatch.go index f89383d1f..e777da587 100644 --- a/acceleration/internal/store/campaign_dispatch.go +++ b/acceleration/internal/store/campaign_dispatch.go @@ -140,18 +140,21 @@ func (s *Store) TakeOverExpired(ctx context.Context, owner string, now time.Time return nil, nil } + expired := s.db.NewSelect().Model((*Contact)(nil)). + Column("cc.id"). + Where("cc.stage = ? AND cc.leased_until < ?", InProgress, now). + Where("NOT EXISTS (SELECT 1 FROM campaigns cp WHERE cp.id = cc.campaign_id AND cp.move_paused)"). + Order("cc.leased_until"). + Limit(limit). + For("UPDATE SKIP LOCKED") var contacts []Contact - err := s.db.NewRaw(`WITH expired AS ( - SELECT id FROM campaign_contacts cc - WHERE stage = ? AND leased_until < ? - AND NOT EXISTS (SELECT 1 FROM campaigns cp WHERE cp.id = cc.campaign_id AND cp.move_paused) - ORDER BY leased_until - LIMIT ? - FOR UPDATE SKIP LOCKED -) -UPDATE campaign_contacts cc SET leased_by = ?, leased_until = ? -FROM expired WHERE cc.id = expired.id -RETURNING cc.*`, InProgress, now, limit, owner, now.Add(lease)).Scan(ctx, &contacts) + err := s.db.NewUpdate().With("expired", expired).Model((*Contact)(nil)). + TableExpr("expired"). + Set("leased_by = ?", owner). + Set("leased_until = ?", now.Add(lease)). + Where("cc.id = expired.id"). + Returning("cc.*"). + Scan(ctx, &contacts) if err != nil { return nil, stack.Wrap(fmt.Errorf("store: take over expired leases: %w", err)) } @@ -457,10 +460,13 @@ func (s *Store) PutOffClassification(ctx context.Context, owner, attemptID strin } var released int64 err := s.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { - result, err := tx.ExecContext(ctx, `UPDATE campaign_contacts cc SET leased_by = '', leased_until = NULL -FROM campaign_attempts ca -WHERE ca.id = ? AND ca.classify_state = ? AND cc.id = ca.contact_id AND cc.stage = ? AND cc.leased_by = ?`, - attemptID, classifyPending, AwaitingOutcome, owner) + result, err := tx.NewUpdate().Model((*Contact)(nil)). + TableExpr("campaign_attempts AS ca"). + Set("leased_by = ''"). + Set("leased_until = NULL"). + Where("ca.id = ? AND ca.classify_state = ?", attemptID, classifyPending). + Where("cc.id = ca.contact_id AND cc.stage = ? AND cc.leased_by = ?", AwaitingOutcome, owner). + Exec(ctx) if err != nil { return fmt.Errorf("store: release a contact to classify later: %w", err) } diff --git a/acceleration/internal/store/image_jobs.go b/acceleration/internal/store/image_jobs.go index c2057453d..039e253f3 100644 --- a/acceleration/internal/store/image_jobs.go +++ b/acceleration/internal/store/image_jobs.go @@ -122,19 +122,20 @@ func (s *Store) ClaimImageJobs(ctx context.Context, now time.Time, limit int, le if limit < 1 { return claimed, nil } - err := s.db.NewRaw(` -WITH due AS ( - SELECT ij.id FROM image_jobs AS ij - WHERE ij.state = 'queued' AND ij.next_attempt_at <= ? AND (ij.message <> '') = ? - ORDER BY ij.next_attempt_at, ij.id - LIMIT ? - FOR UPDATE OF ij SKIP LOCKED -) -UPDATE image_jobs AS ij -SET next_attempt_at = ?, attempts = ij.attempts + 1 -FROM due -WHERE ij.id = due.id -RETURNING `+imageJobClaimed, now.UTC(), decided, limit, leaseUntil.UTC()).Scan(ctx, &claimed) + due := s.db.NewSelect().Model((*ImageJob)(nil)). + Column("ij.id"). + Where("ij.state = 'queued' AND ij.next_attempt_at <= ?", now.UTC()). + Where("(ij.message <> '') = ?", decided). + Order("ij.next_attempt_at", "ij.id"). + Limit(limit). + For("UPDATE OF ij SKIP LOCKED") + err := s.db.NewUpdate().With("due", due).Model((*ImageJob)(nil)). + TableExpr("due"). + Set("next_attempt_at = ?", leaseUntil.UTC()). + Set("attempts = ij.attempts + 1"). + Where("ij.id = due.id"). + Returning(imageJobClaimed). + Scan(ctx, &claimed) if err != nil { return nil, stack.Wrap(fmt.Errorf("store: claim image jobs: %w", err)) } diff --git a/acceleration/internal/store/shipping.go b/acceleration/internal/store/shipping.go index 7dfc110f0..1971d5eb2 100644 --- a/acceleration/internal/store/shipping.go +++ b/acceleration/internal/store/shipping.go @@ -142,9 +142,12 @@ func boundShipping(ctx context.Context, tx bun.Tx) error { func (s *Store) Unshipped(ctx context.Context) (int64, time.Duration, error) { var rows int64 var oldest float64 - err := s.shard.QueryRowContext(ctx, - "SELECT count(*), COALESCE(EXTRACT(EPOCH FROM now() - min(started_at)), 0)::float8 FROM requests WHERE to_ship AND organization_id <> ''", - ).Scan(&rows, &oldest) + err := s.shard.NewSelect().Model((*Request)(nil)). + ColumnExpr("count(*)"). + ColumnExpr("COALESCE(EXTRACT(EPOCH FROM now() - min(r.started_at)), 0)::float8"). + Where("r.to_ship"). + Where("r.organization_id <> ''"). + Scan(ctx, &rows, &oldest) if err != nil { return 0, 0, stack.Wrap(fmt.Errorf("store: unshipped: %w", err)) } @@ -168,21 +171,27 @@ func (s *Store) Reship(ctx context.Context, since time.Time, rejected bool) (int if !last.Valid { return 0, nil } - filter, args := "shipped_at IS NOT NULL AND shipped_at < ?", []any{started} - if rejected { - filter, args = "ship_error <> ''", nil - } var queued int64 for { + batch := s.shard.NewSelect().Model((*Request)(nil)). + Column("r.id"). + Where("r.id > ? AND r.started_at >= ?", last.Int64, since). + Where("r.usage_id IS NOT NULL AND NOT r.to_ship"). + Order("r.id"). + Limit(reshipBatch) + if rejected { + batch.Where("r.ship_error <> ''") + } else { + batch.Where("r.shipped_at IS NOT NULL AND r.shipped_at < ?", started) + } var ids []int64 - err := s.shard.NewRaw(` -WITH batch AS ( - SELECT id FROM requests - WHERE id > ? AND started_at >= ? AND usage_id IS NOT NULL AND NOT to_ship AND `+filter+` - ORDER BY id LIMIT ? -) -UPDATE requests AS r SET to_ship = true, ship_error = '' FROM batch WHERE r.id = batch.id -RETURNING r.id`, append(append([]any{last.Int64, since}, args...), reshipBatch)...).Scan(ctx, &ids) + err := s.shard.NewUpdate().With("batch", batch).Model((*Request)(nil)). + TableExpr("batch"). + Set("to_ship = true"). + Set("ship_error = ''"). + Where("r.id = batch.id"). + Returning("r.id"). + Scan(ctx, &ids) if err != nil { return queued, stack.Wrap(fmt.Errorf("store: reship: %w", err)) } diff --git a/acceleration/internal/store/shipping_test.go b/acceleration/internal/store/shipping_test.go index 123f0a705..036e89f63 100644 --- a/acceleration/internal/store/shipping_test.go +++ b/acceleration/internal/store/shipping_test.go @@ -29,6 +29,76 @@ func (s *StoreSuite) TestMarkingARowAnotherSessionHoldsFailsWithinTheLockWait() s.ErrorContains(err, "lock timeout", "the sweeper's other write waits no longer") } +// shippable records a row of organization's started at at, still to be shipped. +func (s *StoreSuite) shippable(organization string, at time.Time) uuid.UUID { + request := &Request{UsageID: uuid.Must(uuid.NewV7()), Modality: "llm", CustomerID: "acme", OrganizationID: organization, + Provider: "openai", Model: "gpt", StartedAt: at, Success: true, ToShip: true} + s.Require().NoError(s.store.RecordRequest(s.ctx, request)) + return request.UsageID +} + +// shipping is the row's to_ship and ship_error. +func (s *StoreSuite) shipping(usageID uuid.UUID) (bool, string) { + var request Request + s.Require().NoError(s.store.ShardDB().NewSelect().Model(&request).Where("usage_id = ?", usageID).Scan(s.ctx)) + return request.ToShip, request.ShipError +} + +func (s *StoreSuite) TestUnshippedCountsTheRowsWithAnOrganizationStillToShip() { + now := time.Now().UTC() + s.shippable("org", now.Add(-2*time.Hour)) + s.shippable("org", now.Add(-time.Hour)) + s.shippable("", now.Add(-3*time.Hour)) + shipped := s.shippable("org", now.Add(-4*time.Hour)) + s.Require().NoError(s.store.RecordShipment(s.ctx, []uuid.UUID{shipped}, nil)) + + rows, oldest, err := s.store.Unshipped(s.ctx) + + s.Require().NoError(err) + s.EqualValues(2, rows, "neither the shipped row nor the one with no organization") + s.InDelta((2 * time.Hour).Seconds(), oldest.Seconds(), 60) +} + +func (s *StoreSuite) TestReshipQueuesTheShippedRowsStartedSinceAgain() { + since := time.Now().UTC().Add(-time.Hour) + recent := s.shippable("org", since.Add(time.Minute)) + older := s.shippable("org", since.Add(-time.Minute)) + refused := s.shippable("org", since.Add(2*time.Minute)) + waiting := s.shippable("org", since.Add(3*time.Minute)) + s.Require().NoError(s.store.RecordShipment(s.ctx, []uuid.UUID{recent, older}, map[uuid.UUID]string{refused: "unknown price"})) + + queued, err := s.store.Reship(s.ctx, since, false) + + s.Require().NoError(err) + s.EqualValues(1, queued) + toShip, _ := s.shipping(recent) + s.True(toShip) + toShip, _ = s.shipping(older) + s.False(toShip, "started before since") + toShip, reason := s.shipping(refused) + s.False(toShip, "the service refused it, it never had it") + s.Equal("unknown price", reason) + toShip, _ = s.shipping(waiting) + s.True(toShip, "still queued, as it was") +} + +func (s *StoreSuite) TestReshipOfRejectedRowsQueuesTheRefusedOnesAndClearsWhy() { + since := time.Now().UTC().Add(-time.Hour) + shipped := s.shippable("org", since.Add(time.Minute)) + refused := s.shippable("org", since.Add(2*time.Minute)) + s.Require().NoError(s.store.RecordShipment(s.ctx, []uuid.UUID{shipped}, map[uuid.UUID]string{refused: "unknown price"})) + + queued, err := s.store.Reship(s.ctx, since, true) + + s.Require().NoError(err) + s.EqualValues(1, queued) + toShip, reason := s.shipping(refused) + s.True(toShip) + s.Empty(reason) + toShip, _ = s.shipping(shipped) + s.False(toShip, "the service has it") +} + func (s *StoreSuite) TestPostgresProbesTheConnectionHoldingTheShipLock() { lock := s.store.ShipLock() defer lock.Release() From b010bf49bc74cd1b632edab90953eddefea7b7bd Mon Sep 17 00:00:00 2001 From: Thierry Schellenbach Date: Thu, 19 Nov 2026 13:16:48 -0700 Subject: [PATCH 5/7] refactor(store): build reads, forgets and counts on existing models with bun User organizations and settings, linked phones, active price entries, organization apps, tag keys, turns, grant locks, episode summaries, campaign counts and the ledger's forget and prune go through bun's builders. qualified() goes: bun names a model's columns itself. reconcileGrantsQuery stays raw: a test EXPLAINs its text. OrganizationApps had no test; it has now, passing on the old statement too. Co-authored-by: Cursor --- .../internal/store/campaign_analytics.go | 26 +++++++--- acceleration/internal/store/episodes.go | 22 ++++---- acceleration/internal/store/forget.go | 36 +++++++------ acceleration/internal/store/grants.go | 11 ++-- acceleration/internal/store/policies_test.go | 23 ++++++++ acceleration/internal/store/price_changes.go | 8 +-- acceleration/internal/store/prices.go | 23 +++----- acceleration/internal/store/store.go | 52 +++++++++++-------- acceleration/internal/store/user_phones.go | 30 +++++++---- acceleration/internal/usage/ledger/forget.go | 32 +++++++++--- acceleration/internal/usage/ledger/ledger.go | 4 +- 11 files changed, 165 insertions(+), 102 deletions(-) diff --git a/acceleration/internal/store/campaign_analytics.go b/acceleration/internal/store/campaign_analytics.go index 20b08ffb9..7ae6306ab 100644 --- a/acceleration/internal/store/campaign_analytics.go +++ b/acceleration/internal/store/campaign_analytics.go @@ -72,8 +72,12 @@ WHERE cc.campaign_id = ? AND EXISTS (SELECT 1 FROM campaign_attempts ca Outcome string Count int } - err = s.db.NewRaw(`SELECT outcome, count(*) AS count FROM campaign_contacts -WHERE campaign_id = ? AND stage = ? GROUP BY outcome`, campaignID, ContactCompleted).Scan(ctx, &outcomes) + err = s.db.NewSelect().Model((*Contact)(nil)). + Column("cc.outcome"). + ColumnExpr("count(*) AS count"). + Where("cc.campaign_id = ? AND cc.stage = ?", campaignID, ContactCompleted). + Group("cc.outcome"). + Scan(ctx, &outcomes) if err != nil { return CampaignAnalytics{}, stack.Wrap(fmt.Errorf("store: count outcomes: %w", err)) } @@ -85,8 +89,12 @@ WHERE campaign_id = ? AND stage = ? GROUP BY outcome`, campaignID, ContactComple CallOutcome string Count int } - err = s.db.NewRaw(`SELECT call_outcome, count(*) AS count FROM campaign_attempts -WHERE campaign_id = ? AND call_outcome IS NOT NULL GROUP BY call_outcome`, campaignID).Scan(ctx, &calls) + err = s.db.NewSelect().Model((*CampaignAttempt)(nil)). + Column("ca.call_outcome"). + ColumnExpr("count(*) AS count"). + Where("ca.campaign_id = ? AND ca.call_outcome IS NOT NULL", campaignID). + Group("ca.call_outcome"). + Scan(ctx, &calls) if err != nil { return CampaignAnalytics{}, stack.Wrap(fmt.Errorf("store: count call outcomes: %w", err)) } @@ -119,9 +127,13 @@ func (s *Store) ContactStages(ctx context.Context, campaignIDs []string) (map[st Stage string Count int } - err := s.db.NewRaw(`SELECT campaign_id, CASE WHEN stage = ? THEN ? ELSE stage END AS stage, count(*) AS count -FROM campaign_contacts WHERE campaign_id IN (?) GROUP BY 1, 2`, - AwaitingOutcome, InProgress, bun.In(campaignIDs)).Scan(ctx, &rows) + err := s.db.NewSelect().Model((*Contact)(nil)). + Column("cc.campaign_id"). + ColumnExpr("CASE WHEN cc.stage = ? THEN ? ELSE cc.stage END AS stage", AwaitingOutcome, InProgress). + ColumnExpr("count(*) AS count"). + Where("cc.campaign_id IN (?)", bun.In(campaignIDs)). + GroupExpr("1, 2"). + Scan(ctx, &rows) if err != nil { return nil, stack.Wrap(fmt.Errorf("store: count contact stages: %w", err)) } diff --git a/acceleration/internal/store/episodes.go b/acceleration/internal/store/episodes.go index ca37ef86b..159cd8931 100644 --- a/acceleration/internal/store/episodes.go +++ b/acceleration/internal/store/episodes.go @@ -309,24 +309,22 @@ func (s *Store) ClaimEpisodeSummaries(ctx context.Context, now time.Time, limit return claimed, nil } -// finishEpisodeSummaryQuery sets an ended episode's status and drops its lease, in one -// statement. -const finishEpisodeSummaryQuery = ` -WITH finished AS ( - UPDATE episodes SET status = ? - WHERE customer_id = ? AND id = ? AND status = 'ended' - RETURNING id -) -UPDATE episode_activity AS act SET summary_lease_until = NULL -FROM finished WHERE act.episode_id = finished.id` - // FinishEpisodeSummary sets an ended episode summarized or summary_failed and drops its // lease. An episode no longer ended, finished already, is left as it is. func (s *Store) FinishEpisodeSummary(ctx context.Context, customerID, id, status string) error { if status != EpisodeSummarized && status != EpisodeSummaryFailed { return stack.Wrap(fmt.Errorf("store: %q is not a status a summary finishes with", status)) } - _, err := s.db.NewRaw(finishEpisodeSummaryQuery, status, customerID, id).Exec(ctx) + finished := s.db.NewUpdate().Model((*Episode)(nil)). + Set("status = ?", status). + Where("ep.customer_id = ? AND ep.id = ? AND ep.status = ?", customerID, id, EpisodeEnded). + Returning("ep.id") + _, err := s.db.NewUpdate().With("finished", finished). + TableExpr("episode_activity AS act"). + TableExpr("finished"). + Set("summary_lease_until = NULL"). + Where("act.episode_id = finished.id"). + Exec(ctx) if err != nil { return stack.Wrap(fmt.Errorf("store: finish episode summary: %w", err)) } diff --git a/acceleration/internal/store/forget.go b/acceleration/internal/store/forget.go index dfd0ebe5a..d59832605 100644 --- a/acceleration/internal/store/forget.go +++ b/acceleration/internal/store/forget.go @@ -66,8 +66,11 @@ func (s *Store) RenameUserRequests(ctx context.Context, appID, userID, replaceme func (s *Store) UserOrganizations(ctx context.Context, appID string, userIDs ...string) ([]string, error) { shards, err := everyShard(ctx, s, func(ctx context.Context, db bun.IDB) ([]string, error) { var found []string - err := db.NewRaw(`SELECT DISTINCT organization_id FROM requests WHERE customer_id = ? AND user_id IN (?)`, - appID, bun.In(userIDs)).Scan(ctx, &found) + err := db.NewSelect().Model((*Request)(nil)). + Distinct(). + Column("r.organization_id"). + Where("r.customer_id = ? AND r.user_id IN (?)", appID, bun.In(userIDs)). + Scan(ctx, &found) return found, err }) if err != nil { @@ -75,12 +78,14 @@ func (s *Store) UserOrganizations(ctx context.Context, appID string, userIDs ... } var shared []string users := bun.In(userIDs) - if err := s.db.NewRaw(`SELECT organization_id FROM app_organizations WHERE app_id = ? - UNION SELECT organization_id FROM apps WHERE id = ? - UNION SELECT organization_id FROM limits WHERE level = ? AND app_id = ? AND user_id IN (?) - UNION SELECT organization_id FROM usage_resets WHERE app_id = ? AND user_id IN (?) - UNION SELECT organization_id FROM limit_grants WHERE app_id = ? AND user_id IN (?)`, - appID, appID, LevelUser, appID, users, appID, users, appID, users).Scan(ctx, &shared); err != nil { + err = s.db.NewSelect().Model((*AppOrganization)(nil)).Column("organization_id").Where("app_id = ?", appID). + Union(s.db.NewSelect().Model((*App)(nil)).Column("organization_id").Where("id = ?", appID)). + Union(s.db.NewSelect().Model((*Limit)(nil)).Column("organization_id"). + Where("level = ? AND app_id = ? AND user_id IN (?)", LevelUser, appID, users)). + Union(s.db.NewSelect().Model((*UsageReset)(nil)).Column("organization_id").Where("app_id = ? AND user_id IN (?)", appID, users)). + Union(s.db.NewSelect().Model((*LimitGrant)(nil)).Column("organization_id").Where("app_id = ? AND user_id IN (?)", appID, users)). + Scan(ctx, &shared) + if err != nil { return nil, stack.Wrap(fmt.Errorf("store: user organizations: %w", err)) } organizations := map[string]bool{} @@ -121,16 +126,15 @@ func (s *Store) ForgetUserSettings(ctx context.Context, appID, userID, replaceme err := s.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { for _, change := range []struct { count *int64 - query string - args []any + exec func(context.Context, ...any) (sql.Result, error) }{ - {&counts.Policies, "DELETE FROM policies WHERE scope = ? AND scope_id = ?", []any{ScopeUser, UserScopeID(appID, userID)}}, - {&counts.Limits, "DELETE FROM limits WHERE level = ? AND app_id = ? AND user_id = ?", []any{LevelUser, appID, userID}}, - {&counts.Resets, "UPDATE usage_resets SET user_id = ? WHERE app_id = ? AND user_id = ?", []any{replacement, appID, userID}}, - {&counts.Grants, "UPDATE limit_grants SET user_id = ? WHERE app_id = ? AND user_id = ?", []any{replacement, appID, userID}}, - {&counts.Campaigns, "UPDATE campaigns SET user_id = ? WHERE customer_id = ? AND user_id = ?", []any{replacement, appID, userID}}, + {&counts.Policies, tx.NewDelete().Model((*Policy)(nil)).Where("scope = ? AND scope_id = ?", ScopeUser, UserScopeID(appID, userID)).Exec}, + {&counts.Limits, tx.NewDelete().Model((*Limit)(nil)).Where("level = ? AND app_id = ? AND user_id = ?", LevelUser, appID, userID).Exec}, + {&counts.Resets, tx.NewUpdate().Model((*UsageReset)(nil)).Set("user_id = ?", replacement).Where("app_id = ? AND user_id = ?", appID, userID).Exec}, + {&counts.Grants, tx.NewUpdate().Model((*LimitGrant)(nil)).Set("user_id = ?", replacement).Where("app_id = ? AND user_id = ?", appID, userID).Exec}, + {&counts.Campaigns, tx.NewUpdate().Model((*Campaign)(nil)).Set("user_id = ?", replacement).Where("customer_id = ? AND user_id = ?", appID, userID).Exec}, } { - result, err := tx.ExecContext(ctx, change.query, change.args...) + result, err := change.exec(ctx) if err != nil { return err } diff --git a/acceleration/internal/store/grants.go b/acceleration/internal/store/grants.go index 572671c64..03e5ee4de 100644 --- a/acceleration/internal/store/grants.go +++ b/acceleration/internal/store/grants.go @@ -402,12 +402,17 @@ func addGrantUses(ctx context.Context, tx bun.Tx, uses []GrantUse) (int, error) for i, use := range uses { grants[i], usageIDs[i], micros[i] = use.GrantID, use.UsageID.String(), use.Micros } - if _, err := tx.ExecContext(ctx, "SELECT id FROM limit_grants WHERE id = ANY(?::bigint[]) ORDER BY id FOR UPDATE", - pgdialect.Array(slices.Compact(slices.Sorted(slices.Values(grants))))); err != nil { + _, err := tx.NewSelect().Model((*LimitGrant)(nil)). + Column("lg.id"). + Where("lg.id = ANY(?::bigint[])", pgdialect.Array(slices.Compact(slices.Sorted(slices.Values(grants))))). + Order("lg.id"). + For("UPDATE"). + Exec(ctx) + if err != nil { return 0, err } var added int - err := tx.NewRaw(` + err = tx.NewRaw(` WITH sent AS ( SELECT * FROM unnest(?::bigint[], ?::uuid[], ?::bigint[]) AS s (grant_id, usage_id, micros) ), counted AS ( diff --git a/acceleration/internal/store/policies_test.go b/acceleration/internal/store/policies_test.go index 666be8954..9071a1180 100644 --- a/acceleration/internal/store/policies_test.go +++ b/acceleration/internal/store/policies_test.go @@ -3,6 +3,7 @@ package store import ( + "slices" "time" "github.com/google/uuid" @@ -155,6 +156,28 @@ func (s *StoreSuite) TestAnAppIsInTheOrganizationItIsRegisteredOrSeenUnder() { } } +func (s *StoreSuite) TestAnOrganizationsAppsAreTheOnesSeenOrRegisteredUnderItAndItsOwn() { + org := Organization{Name: "registering"} + s.Require().NoError(s.store.CreateOrganization(s.ctx, &org)) + registered := App{OrganizationID: org.ID, Name: "registered"} + s.Require().NoError(s.store.CreateApp(s.ctx, ®istered)) + moved := App{OrganizationID: org.ID, Name: "moved"} + s.Require().NoError(s.store.CreateApp(s.ctx, &moved)) + s.Require().NoError(s.store.JoinOrganization(s.ctx, moved.ID, "org-other")) + s.Require().NoError(s.store.JoinOrganization(s.ctx, "app-seen-b", org.ID)) + s.Require().NoError(s.store.JoinOrganization(s.ctx, "app-seen-a", org.ID)) + s.Require().NoError(s.store.JoinOrganization(s.ctx, "app-elsewhere", "org-other")) + + apps, err := s.store.OrganizationApps(s.ctx, org.ID) + + s.Require().NoError(err) + s.Require().NotEmpty(apps) + s.Equal(org.ID, apps[0], "its own id first") + s.ElementsMatch([]string{org.ID, registered.ID, "app-seen-a", "app-seen-b"}, apps, + "not the app a request named another organization for, nor another organization's") + s.Less(slices.Index(apps, "app-seen-a"), slices.Index(apps, "app-seen-b"), "in order") +} + func scopeIDs(policies []Policy) []string { ids := make([]string, len(policies)) for i, policy := range policies { diff --git a/acceleration/internal/store/price_changes.go b/acceleration/internal/store/price_changes.go index b8ef9d895..42e0bd717 100644 --- a/acceleration/internal/store/price_changes.go +++ b/acceleration/internal/store/price_changes.go @@ -258,8 +258,8 @@ func (s *Store) CreatePriceChange(ctx context.Context, write PriceChangeWrite, return err } } - var seeded bool - if err := tx.QueryRowContext(ctx, "SELECT EXISTS (SELECT 1 FROM price_changes)").Scan(&seeded); err != nil { + seeded, err := tx.NewSelect().Model((*PriceChange)(nil)).Exists(ctx) + if err != nil { return err } if !seeded { @@ -289,8 +289,8 @@ func (s *Store) CreatePriceChange(ctx context.Context, write PriceChangeWrite, CreatedBy: write.ActorID, IdempotencyKey: write.IdempotencyKey, } - var taken bool - if err := tx.QueryRowContext(ctx, "SELECT EXISTS (SELECT 1 FROM price_changes WHERE name = ?)", change.Name).Scan(&taken); err != nil { + taken, err := tx.NewSelect().Model((*PriceChange)(nil)).Where("name = ?", change.Name).Exists(ctx) + if err != nil { return err } if taken { diff --git a/acceleration/internal/store/prices.go b/acceleration/internal/store/prices.go index b2d5a1279..60939a043 100644 --- a/acceleration/internal/store/prices.go +++ b/acceleration/internal/store/prices.go @@ -417,13 +417,13 @@ func (s *Store) ActivePriceEntries(ctx context.Context, at time.Time) ([]PriceEn func activePriceEntries(ctx context.Context, db bun.IDB, at time.Time) ([]PriceEntry, error) { var newest []PriceEntry - err := db.NewRaw(` -SELECT DISTINCT ON (pe.modality, pe.provider, pe.model, pe.region, pe.byok, pe.attributes) `+qualified("pe", priceEntryColumns)+` -FROM price_list_entries AS pe -JOIN price_changes AS pc ON pc.name = pe.change AND pc.cancelled_at IS NULL -WHERE pe.effective_at <= ? -ORDER BY pe.modality, pe.provider, pe.model, pe.region, pe.byok, pe.attributes, pe.effective_at DESC, pe.change DESC, pe.id DESC`, at). - Scan(ctx, &newest) + err := db.NewSelect().Model(&newest). + DistinctOn("pe.modality, pe.provider, pe.model, pe.region, pe.byok, pe.attributes"). + Join("JOIN price_changes AS pc ON pc.name = pe.change AND pc.cancelled_at IS NULL"). + Where("pe.effective_at <= ?", at). + Order("pe.modality", "pe.provider", "pe.model", "pe.region", "pe.byok", "pe.attributes"). + OrderExpr("pe.effective_at DESC, pe.change DESC, pe.id DESC"). + Scan(ctx) if err != nil { return nil, stack.Wrap(fmt.Errorf("store: active price entries: %w", err)) } @@ -436,15 +436,6 @@ ORDER BY pe.modality, pe.provider, pe.model, pe.region, pe.byok, pe.attributes, return active, nil } -// qualified is a list of columns each prefixed with the table's alias. -func qualified(alias, columns string) string { - names := strings.Split(columns, ", ") - for i, name := range names { - names[i] = alias + "." + name - } - return strings.Join(names, ", ") -} - // NextPriceChangeAt is when the next change not cancelled takes effect after after, and // whether one is scheduled, so a list in memory can switch at that moment. func (s *Store) NextPriceChangeAt(ctx context.Context, after time.Time) (time.Time, bool, error) { diff --git a/acceleration/internal/store/store.go b/acceleration/internal/store/store.go index 4a48acbfc..9870fe234 100644 --- a/acceleration/internal/store/store.go +++ b/acceleration/internal/store/store.go @@ -876,12 +876,16 @@ func spendGroups(groupBy string, acrossApps bool) (value, join string, ok bool) func (s *Store) OrganizationApps(ctx context.Context, organizationID string) ([]string, error) { // One statement, so an app a request records under the organization meanwhile is in one // half or the other. + namedOther := s.db.NewSelect().Model((*AppOrganization)(nil)). + ColumnExpr("1"). + Where("ao.app_id = ap.id AND ao.organization_id <> ?", organizationID) + named := s.db.NewSelect().Model((*AppOrganization)(nil)).Column("app_id").Where("organization_id = ?", organizationID). + Union(s.db.NewSelect().Model((*App)(nil)). + Column("id"). + Where("ap.organization_id = ?", organizationID). + Where("NOT EXISTS (?)", namedOther)) var members []string - err := s.db.NewRaw(`SELECT app_id FROM app_organizations WHERE organization_id = ? -UNION -SELECT id FROM apps AS ap WHERE organization_id = ? - AND NOT EXISTS (SELECT 1 FROM app_organizations AS ao WHERE ao.app_id = ap.id AND ao.organization_id <> ?) -ORDER BY 1`, organizationID, organizationID, organizationID).Scan(ctx, &members) + err := s.db.NewSelect().TableExpr("(?) AS member", named).Column("member.app_id").OrderExpr("1").Scan(ctx, &members) if err != nil { return nil, stack.Wrap(fmt.Errorf("store: organization apps: %w", err)) } @@ -1092,15 +1096,16 @@ func (s *Store) CustomerTagKeys( if err != nil || read.rows == 0 { return read, err } - err = db.NewRaw(` -SELECT tag.key AS key, tag.value AS value, - SUM(r.cost_micros) AS cost_micros_total, `+requestCount+` AS request_count -FROM requests AS r -CROSS JOIN LATERAL jsonb_each_text(r.tags) AS tag(key, value) -WHERE r.customer_id = ? - AND r.started_at >= ? AND r.started_at < ? - AND r.tags @> ?::jsonb -GROUP BY tag.key, tag.value`, customerID, from, to, string(filter)).Scan(ctx, &read.values) + err = db.NewSelect().Model((*Request)(nil)). + ColumnExpr("tag.key AS key, tag.value AS value"). + ColumnExpr("SUM(r.cost_micros) AS cost_micros_total"). + ColumnExpr(requestCount+" AS request_count"). + Join("CROSS JOIN LATERAL jsonb_each_text(r.tags) AS tag(key, value)"). + Where("r.customer_id = ?", customerID). + Where("r.started_at >= ? AND r.started_at < ?", from, to). + Where("r.tags @> ?::jsonb", string(filter)). + Group("tag.key", "tag.value"). + Scan(ctx, &read.values) return read, err }) if err != nil { @@ -1606,15 +1611,16 @@ func (s *Store) CustomerTurns(ctx context.Context, customerID, configID string, } counts, err := everyShard(ctx, s, func(ctx context.Context, db bun.IDB) (SessionTurns, error) { var counts SessionTurns - err := db.NewRaw(` -SELECT COUNT(DISTINCT a.session_id) AS sessions, COUNT(*) AS turns -FROM agent_responses AS a -JOIN agent_sessions AS s ON s.id = a.session_id -WHERE a.customer_id = ?0 AND (?1 = '' OR s.config_id = ?1) - AND a.created_at >= ?2 AND a.created_at < ?3 - AND a.parent_response_id IS NULL AND a.rewound_at IS NULL`, - customerID, configID, from, to, - ).Scan(ctx, &counts) + q := db.NewSelect().Model((*AgentResponse)(nil)). + ColumnExpr("COUNT(DISTINCT ars.session_id) AS sessions, COUNT(*) AS turns"). + Join("JOIN agent_sessions AS asn ON asn.id = ars.session_id"). + Where("ars.customer_id = ?", customerID). + Where("ars.created_at >= ? AND ars.created_at < ?", from, to). + Where("ars.parent_response_id IS NULL AND ars.rewound_at IS NULL") + if configID != "" { + q.Where("asn.config_id = ?", configID) + } + err := q.Scan(ctx, &counts) return counts, err }) if err != nil { diff --git a/acceleration/internal/store/user_phones.go b/acceleration/internal/store/user_phones.go index 6caf62340..e105d8f77 100644 --- a/acceleration/internal/store/user_phones.go +++ b/acceleration/internal/store/user_phones.go @@ -178,8 +178,12 @@ func (s *Store) LinkedPhone(ctx context.Context, customerID string, app int64, u Phone string `bun:"phone"` Since *time.Time `bun:"since"` } - err := s.db.NewRaw(`SELECT up.phone, `+historyBoundary+` AS since FROM user_phones AS up `+historyJoin+` -WHERE up.customer_id = ? AND up.stream_app_pk = ? AND up.user_id = ?`, customerID, s.linkApp(app), userID).Scan(ctx, &rows) + err := s.db.NewSelect().Model((*UserPhone)(nil)). + Column("up.phone"). + ColumnExpr(historyBoundary+" AS since"). + Join(historyJoin). + Where("up.customer_id = ? AND up.stream_app_pk = ? AND up.user_id = ?", customerID, s.linkApp(app), userID). + Scan(ctx, &rows) if err != nil { return "", nil, false, stack.Wrap(fmt.Errorf("store: linked phone: %w", err)) } @@ -411,15 +415,19 @@ const reachedAt = `regexp_replace(cm.address, '^` + retiredPrefix + `[^:]*:', '' // counts as on: it writes the same episodes, which a session reads once the config turns on. // In tx, under the link's lock. func (s *Store) retireHistory(ctx context.Context, tx bun.Tx, link UserPhone) error { - _, err := tx.NewRaw(`UPDATE contact_map AS cm -SET address = ?4 || cm.id || ':' || cm.address, updated_at = ?5 -WHERE cm.customer_id = ?0 - AND cm.agent_config_id IN (SELECT ac.id FROM agent_configs AS ac WHERE ac.customer_id = ?0 AND (ac.episode_cards OR ac.episode_cards_record)) - AND cm.kind = ?1 AND cm.address = ?2 - AND COALESCE(NULLIF(cm.stream_app_pk, ?3), 0) = ?6 - AND EXISTS (SELECT 1 FROM episodes AS ep WHERE ep.contact_id = cm.id)`, - link.CustomerID, ContactPhone, link.Phone, s.streamPins().deployment(), retiredPrefix, time.Now().UTC(), - link.StreamAppPK). + writesCards := tx.NewSelect().Model((*AgentConfig)(nil)). + Column("ac.id"). + Where("ac.customer_id = ?", link.CustomerID). + Where("ac.episode_cards OR ac.episode_cards_record") + withEpisode := tx.NewSelect().Model((*Episode)(nil)).ColumnExpr("1").Where("ep.contact_id = cm.id") + _, err := tx.NewUpdate().Model((*ContactMapEntry)(nil)). + Set("address = ? || cm.id || ':' || cm.address", retiredPrefix). + Set("updated_at = ?", time.Now().UTC()). + Where("cm.customer_id = ?", link.CustomerID). + Where("cm.agent_config_id IN (?)", writesCards). + Where("cm.kind = ? AND cm.address = ?", ContactPhone, link.Phone). + Where("COALESCE(NULLIF(cm.stream_app_pk, ?), 0) = ?", s.streamPins().deployment(), link.StreamAppPK). + Where("EXISTS (?)", withEpisode). Exec(ctx) if err != nil { return stack.Wrap(fmt.Errorf("store: retire a linked number's history: %w", err)) diff --git a/acceleration/internal/usage/ledger/forget.go b/acceleration/internal/usage/ledger/forget.go index 4ec298f6c..eb5161f80 100644 --- a/acceleration/internal/usage/ledger/forget.go +++ b/acceleration/internal/usage/ledger/forget.go @@ -25,17 +25,33 @@ func (l *Ledger) ForgetUser(ctx context.Context, forget usage.ForgetUser, actor var events, windows []counted err := l.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { if forget.DryRun { - if err := tx.NewRaw(`SELECT organization_id, count(*) FROM usage_events WHERE app_id = ? AND user_id = ? - GROUP BY organization_id`, forget.AppID, forget.UserID).Scan(ctx, &events); err != nil { + err := tx.NewSelect().Model((*UsageEvent)(nil)). + Column("organization_id"). + ColumnExpr("count(*) AS count"). + Where("app_id = ? AND user_id = ?", forget.AppID, forget.UserID). + Group("organization_id"). + Scan(ctx, &events) + if err != nil { return err } - return tx.NewRaw(`SELECT organization_id, count(*) FROM usage_windows WHERE subject = ? AND app_id = ? AND user_id = ? - GROUP BY organization_id`, usage.SubjectUser, forget.AppID, forget.UserID).Scan(ctx, &windows) + return tx.NewSelect().Model((*UsageWindow)(nil)). + Column("organization_id"). + ColumnExpr("count(*) AS count"). + Where("subject = ? AND app_id = ? AND user_id = ?", usage.SubjectUser, forget.AppID, forget.UserID). + Group("organization_id"). + Scan(ctx, &windows) } - if err := tx.NewRaw(`WITH moved AS ( - UPDATE usage_events SET user_id = ? WHERE app_id = ? AND user_id = ? RETURNING organization_id) - SELECT organization_id, count(*) FROM moved GROUP BY organization_id`, - forget.Replacement, forget.AppID, forget.UserID).Scan(ctx, &events); err != nil { + moved := tx.NewUpdate().Model((*UsageEvent)(nil)). + Set("user_id = ?", forget.Replacement). + Where("app_id = ? AND user_id = ?", forget.AppID, forget.UserID). + Returning("organization_id") + err := tx.NewSelect().With("moved", moved). + Table("moved"). + Column("organization_id"). + ColumnExpr("count(*) AS count"). + Group("organization_id"). + Scan(ctx, &events) + if err != nil { return err } if err := tx.NewRaw(`WITH moved AS ( diff --git a/acceleration/internal/usage/ledger/ledger.go b/acceleration/internal/usage/ledger/ledger.go index 55005fd11..e7aad4a8e 100644 --- a/acceleration/internal/usage/ledger/ledger.go +++ b/acceleration/internal/usage/ledger/ledger.go @@ -383,8 +383,8 @@ func storedWindows(ctx context.Context, db bun.IDB, wanted []UsageWindow) ([]Usa func (l *Ledger) PruneWindows(ctx context.Context, cutoff time.Time) (int64, error) { var deleted int64 for { - result, err := l.db.NewRaw(`DELETE FROM usage_windows WHERE ctid IN ( - SELECT ctid FROM usage_windows WHERE window_end < ? LIMIT ?)`, cutoff, pruneBatch).Exec(ctx) + ended := l.db.NewSelect().Model((*UsageWindow)(nil)).Column("ctid").Where("window_end < ?", cutoff).Limit(pruneBatch) + result, err := l.db.NewDelete().Model((*UsageWindow)(nil)).Where("ctid IN (?)", ended).Exec(ctx) if err != nil { return deleted, fmt.Errorf("ledger: prune windows: %w", err) } From 79f7dd341f780146530328309f6f44bab97abcdc Mon Sep 17 00:00:00 2001 From: Thierry Schellenbach Date: Thu, 19 Nov 2026 13:36:13 -0700 Subject: [PATCH 6/7] fix(store): keep the channel inbound queue insert raw bun writes a JSON \u0000 as the text \\u0000, so a delivery whose headers jsonb cannot hold was queued and claimed instead of failing, and the provider's retry found it taken. The insert goes back to json.Marshal and ?::jsonb, which TestAMessageTheStoreFailsToQueueIsNotClaimed checks. Co-authored-by: Cursor --- .../internal/store/channel_inbound.go | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/acceleration/internal/store/channel_inbound.go b/acceleration/internal/store/channel_inbound.go index d4c85ce5e..136d9163d 100644 --- a/acceleration/internal/store/channel_inbound.go +++ b/acceleration/internal/store/channel_inbound.go @@ -215,12 +215,28 @@ func queueChannelInbound(ctx context.Context, tx bun.Tx, message ChannelInbound) if _, err := tx.NewRaw("SELECT 1 FROM channel_threads WHERE channel_id = ? FOR NO KEY UPDATE", message.ChannelID).Exec(ctx); err != nil { return false, stack.Wrap(fmt.Errorf("store: queue channel inbound: %w", err)) } - message.NextAttemptAt = message.NextAttemptAt.UTC() - result, err := tx.NewInsert().Model(&message). - Column("channel_id", "provider_message_id", "customer_id", "connector_id", "provider_unit_id", "thread_key", - "config_id", "config_name", "author_id", "text", "media", "raw", "forward", "next_attempt_at"). - On("CONFLICT (channel_id, provider_message_id) DO NOTHING"). - Exec(ctx) + // Not the insert builder: bun writes a JSON \u0000 as the text \\u0000, and a delivery jsonb + // cannot hold must fail here, so it stays unclaimed and the provider's retry takes it again. + media, err := json.Marshal(message.Media) + if err != nil { + return false, stack.Wrap(err) + } + var forward *string + if message.Forward != nil { + encoded, err := json.Marshal(message.Forward) + if err != nil { + return false, stack.Wrap(err) + } + forward = new(string(encoded)) + } + result, err := tx.NewRaw(` +INSERT INTO channel_thread_inbound (channel_id, provider_message_id, customer_id, connector_id, provider_unit_id, thread_key, + config_id, config_name, author_id, text, media, raw, forward, next_attempt_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::jsonb, ?, ?::jsonb, ?) +ON CONFLICT (channel_id, provider_message_id) DO NOTHING`, + message.ChannelID, message.ProviderMessageID, message.CustomerID, message.ConnectorID, message.ProviderUnitID, message.ThreadKey, + message.ConfigID, message.ConfigName, message.AuthorID, message.Text, string(media), message.Raw, forward, + message.NextAttemptAt.UTC()).Exec(ctx) if err != nil { return false, stack.Wrap(fmt.Errorf("store: queue channel inbound: %w", err)) } From ebaf4dd3c4f893c9d55cb731515fcbdecc82ac24 Mon Sep 17 00:00:00 2001 From: Thierry Schellenbach Date: Thu, 19 Nov 2026 13:36:13 -0700 Subject: [PATCH 7/7] refactor(store): give small tables models and use bun's builders channel_thread_*, channel_deliveries, chat_channel_turns, connection_event_deliveries, hook_deliveries, data_move_cursors and limit_grant_uses/reconciles get models, and their statements move to the insert, select and delete builders. Tests cover GrantUsesCounted and SetChannelThreadStarter, which none did; both pass on the raw SQL too. Co-authored-by: Cursor --- .../internal/store/channel_threads.go | 128 ++++++++++++------ .../internal/store/channel_threads_test.go | 15 ++ acceleration/internal/store/channels.go | 28 ++-- .../internal/store/chat_channel_turns.go | 29 ++-- .../internal/store/connection_events.go | 16 ++- acceleration/internal/store/grants.go | 38 +++++- acceleration/internal/store/grants_test.go | 16 +++ acceleration/internal/store/hooks.go | 17 ++- acceleration/internal/store/movecursor.go | 30 ++-- 9 files changed, 240 insertions(+), 77 deletions(-) diff --git a/acceleration/internal/store/channel_threads.go b/acceleration/internal/store/channel_threads.go index 5e6da4ea4..93ebfe625 100644 --- a/acceleration/internal/store/channel_threads.go +++ b/acceleration/internal/store/channel_threads.go @@ -99,11 +99,48 @@ func (s *Store) ReleaseChannelThreadTurn(ctx context.Context, channelID, holder return stack.Wrap(err) } +// channelThreadMessage is one claimed message of a thread channel (ClaimChannelThreadMessage). +type channelThreadMessage struct { + bun.BaseModel `bun:"table:channel_thread_messages,alias:ctmsg"` + + ChannelID string `bun:"channel_id,pk"` + Kind string `bun:"kind,pk"` + MessageID string `bun:"message_id,pk"` + CreatedAt time.Time `bun:"created_at,notnull"` +} + +// channelThreadPerson is the one person a thread channel's chat is with +// (SetChannelThreadChatPerson). +type channelThreadPerson struct { + bun.BaseModel `bun:"table:channel_thread_people,alias:ctp"` + + ChannelID string `bun:"channel_id,pk"` + ChatPerson string `bun:"chat_person,notnull"` +} + +// channelThreadMember is a person a thread channel's chat reaches (AddChannelThreadMember). +type channelThreadMember struct { + bun.BaseModel `bun:"table:channel_thread_members,alias:ctm"` + + ChannelID string `bun:"channel_id,pk"` + Member string `bun:"member,pk"` +} + +// channelThreadStarter is the person who started a thread channel's chat +// (SetChannelThreadStarter). +type channelThreadStarter struct { + bun.BaseModel `bun:"table:channel_thread_starters,alias:cts"` + + ChannelID string `bun:"channel_id,pk"` + Starter string `bun:"starter,notnull"` +} + // ReleaseChannelThreadMessage forgets a claim, so the step that took it can take it again: // a reply whose send failed is sent on a later attempt. func (s *Store) ReleaseChannelThreadMessage(ctx context.Context, channelID, kind, messageID string) error { - _, err := s.db.NewRaw("DELETE FROM channel_thread_messages WHERE channel_id = ? AND kind = ? AND message_id = ?", - channelID, kind, messageID).Exec(ctx) + _, err := s.db.NewDelete().Model((*channelThreadMessage)(nil)). + Where("channel_id = ? AND kind = ? AND message_id = ?", channelID, kind, messageID). + Exec(ctx) return stack.Wrap(err) } @@ -143,10 +180,12 @@ func (s *Store) SetChannelThreadChatPerson(ctx context.Context, channelID, perso } var err error if person == "" { - _, err = s.db.NewRaw("DELETE FROM channel_thread_people WHERE channel_id = ?", channelID).Exec(ctx) + _, err = s.db.NewDelete().Model((*channelThreadPerson)(nil)).Where("channel_id = ?", channelID).Exec(ctx) } else { - _, err = s.db.NewRaw("INSERT INTO channel_thread_people (channel_id, chat_person) VALUES (?, ?) "+ - "ON CONFLICT (channel_id) DO UPDATE SET chat_person = EXCLUDED.chat_person", channelID, person).Exec(ctx) + _, err = s.db.NewInsert().Model(&channelThreadPerson{ChannelID: channelID, ChatPerson: person}). + On("CONFLICT (channel_id) DO UPDATE"). + Set("chat_person = EXCLUDED.chat_person"). + Exec(ctx) } if err != nil { return stack.Wrap(fmt.Errorf("store: set channel thread chat person: %w", err)) @@ -161,8 +200,9 @@ func (s *Store) AddChannelThreadMember(ctx context.Context, channelID, member st if channelID == "" || member == "" { return stack.Wrap(errors.New("store: a channel id and a member are required")) } - _, err := s.db.NewRaw("INSERT INTO channel_thread_members (channel_id, member) VALUES (?, ?) ON CONFLICT DO NOTHING", - channelID, member).Exec(ctx) + _, err := s.db.NewInsert().Model(&channelThreadMember{ChannelID: channelID, Member: member}). + On("CONFLICT DO NOTHING"). + Exec(ctx) if err != nil { return stack.Wrap(fmt.Errorf("store: add channel thread member: %w", err)) } @@ -177,8 +217,9 @@ func (s *Store) SetChannelThreadStarter(ctx context.Context, channelID, starter if channelID == "" || starter == "" { return stack.Wrap(errors.New("store: a channel id and a starter are required")) } - _, err := s.db.NewRaw("INSERT INTO channel_thread_starters (channel_id, starter) VALUES (?, ?) ON CONFLICT DO NOTHING", - channelID, starter).Exec(ctx) + _, err := s.db.NewInsert().Model(&channelThreadStarter{ChannelID: channelID, Starter: starter}). + On("CONFLICT DO NOTHING"). + Exec(ctx) if err != nil { return stack.Wrap(fmt.Errorf("store: set channel thread starter: %w", err)) } @@ -189,7 +230,10 @@ func (s *Store) SetChannelThreadStarter(ctx context.Context, channelID, starter // (SetChannelThreadStarter), or "" when none is recorded. func (s *Store) ChannelThreadStarter(ctx context.Context, channelID string) (string, error) { var starter string - err := s.db.NewRaw("SELECT starter FROM channel_thread_starters WHERE channel_id = ?", channelID).Scan(ctx, &starter) + err := s.db.NewSelect().Model((*channelThreadStarter)(nil)). + Column("starter"). + Where("channel_id = ?", channelID). + Scan(ctx, &starter) if errors.Is(err, sql.ErrNoRows) { return "", nil } @@ -252,14 +296,17 @@ func (s *Store) ChannelThread(ctx context.Context, channelID string) (ChannelThr // waitingMessage is one row of channel_thread_waiting (20261011220000_channel_thread_waiting.sql). type waitingMessage struct { - ConnectorID string `bun:"connector_id"` - ProviderUnitID string `bun:"provider_unit_id"` - ThreadKey string `bun:"thread_key"` - ProviderMessageID string `bun:"provider_message_id"` - AuthorID string `bun:"author_id"` - Text string `bun:"text"` - Raw []byte `bun:"raw"` - CreatedAt time.Time `bun:"created_at"` + bun.BaseModel `bun:"table:channel_thread_waiting,alias:ctw"` + + CustomerID string `bun:"customer_id,pk"` + ConnectorID string `bun:"connector_id,pk"` + ProviderUnitID string `bun:"provider_unit_id,pk"` + ThreadKey string `bun:"thread_key,pk"` + ProviderMessageID string `bun:"provider_message_id,pk"` + AuthorID string `bun:"author_id,notnull"` + Text string `bun:"text,notnull"` + Raw []byte `bun:"raw,notnull"` + CreatedAt time.Time `bun:"created_at,notnull"` } // WaitChannelThreadMessage keeps a customer's message on an external thread no thread channel @@ -271,14 +318,15 @@ func (s *Store) WaitChannelThreadMessage(ctx context.Context, customerID string, return false, stack.Wrap(errors.New("store: a customer, a connector, a thread key and a message id are required")) } now := time.Now().UTC() - if _, err := s.db.NewRaw("DELETE FROM channel_thread_waiting WHERE created_at < ?", now.Add(-channelWaitingKeep)).Exec(ctx); err != nil { + _, err := s.db.NewDelete().Model((*waitingMessage)(nil)).Where("created_at < ?", now.Add(-channelWaitingKeep)).Exec(ctx) + if err != nil { return false, stack.Wrap(fmt.Errorf("store: wait channel thread message: %w", err)) } - result, err := s.db.NewRaw( - "INSERT INTO channel_thread_waiting (customer_id, connector_id, provider_unit_id, thread_key, provider_message_id, author_id, text, raw, created_at) "+ - "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING", - customerID, message.ConnectorID, message.ProviderUnitID, message.ThreadKey, message.ProviderMessageID, - message.AuthorID, message.Text, message.Raw, now).Exec(ctx) + result, err := s.db.NewInsert().Model(&waitingMessage{ + CustomerID: customerID, ConnectorID: message.ConnectorID, ProviderUnitID: message.ProviderUnitID, + ThreadKey: message.ThreadKey, ProviderMessageID: message.ProviderMessageID, + AuthorID: message.AuthorID, Text: message.Text, Raw: message.Raw, CreatedAt: now, + }).On("CONFLICT DO NOTHING").Exec(ctx) if err != nil { return false, stack.Wrap(fmt.Errorf("store: wait channel thread message: %w", err)) } @@ -292,9 +340,11 @@ func (s *Store) WaitChannelThreadMessage(ctx context.Context, customerID string, // UnwaitChannelThreadMessage takes one waiting message back. False is one that is not waiting: // the message that linked its thread took it first, or it was never kept. func (s *Store) UnwaitChannelThreadMessage(ctx context.Context, customerID string, message core.InboundMessage) (bool, error) { - result, err := s.db.NewRaw( - "DELETE FROM channel_thread_waiting WHERE customer_id = ? AND connector_id = ? AND provider_unit_id = ? AND thread_key = ? AND provider_message_id = ?", - customerID, message.ConnectorID, message.ProviderUnitID, message.ThreadKey, message.ProviderMessageID).Exec(ctx) + result, err := s.db.NewDelete().Model((*waitingMessage)(nil)). + Where("customer_id = ? AND connector_id = ? AND provider_unit_id = ? AND thread_key = ?", + customerID, message.ConnectorID, message.ProviderUnitID, message.ThreadKey). + Where("provider_message_id = ?", message.ProviderMessageID). + Exec(ctx) if err != nil { return false, stack.Wrap(fmt.Errorf("store: unwait channel thread message: %w", err)) } @@ -311,10 +361,12 @@ func (s *Store) UnwaitChannelThreadMessage(ctx context.Context, customerID strin // (UnwaitChannelThreadMessage) gets it, or this does. func takeWaitingChannelThreadMessages(ctx context.Context, db bun.IDB, customerID, connectorID, providerUnitID, threadKey string) ([]core.InboundMessage, error) { var rows []waitingMessage - err := db.NewRaw( - "DELETE FROM channel_thread_waiting WHERE customer_id = ? AND connector_id = ? AND provider_unit_id = ? AND thread_key = ? AND created_at >= ? "+ - "RETURNING connector_id, provider_unit_id, thread_key, provider_message_id, author_id, text, raw, created_at", - customerID, connectorID, providerUnitID, threadKey, time.Now().UTC().Add(-channelWaitingKeep)).Scan(ctx, &rows) + err := db.NewDelete().Model((*waitingMessage)(nil)). + Where("customer_id = ? AND connector_id = ? AND provider_unit_id = ? AND thread_key = ?", + customerID, connectorID, providerUnitID, threadKey). + Where("created_at >= ?", time.Now().UTC().Add(-channelWaitingKeep)). + Returning("connector_id, provider_unit_id, thread_key, provider_message_id, author_id, text, raw, created_at"). + Scan(ctx, &rows) if err != nil { return nil, stack.Wrap(fmt.Errorf("store: take waiting channel thread messages: %w", err)) } @@ -355,15 +407,15 @@ func claimChannelThreadMessage(ctx context.Context, db bun.IDB, channelID, kind, return false, stack.Wrap(errors.New("store: a channel, a kind and a message id are required")) } now := time.Now().UTC() - if _, err := db.NewRaw( - "DELETE FROM channel_thread_messages WHERE channel_id = ? AND created_at < ?", - channelID, now.Add(-channelMessageKeep)).Exec(ctx); err != nil { + _, err := db.NewDelete().Model((*channelThreadMessage)(nil)). + Where("channel_id = ? AND created_at < ?", channelID, now.Add(-channelMessageKeep)). + Exec(ctx) + if err != nil { return false, stack.Wrap(fmt.Errorf("store: claim channel thread message: %w", err)) } - result, err := db.NewRaw( - "INSERT INTO channel_thread_messages (channel_id, kind, message_id, created_at) "+ - "VALUES (?, ?, ?, ?) ON CONFLICT DO NOTHING", - channelID, kind, messageID, now).Exec(ctx) + result, err := db.NewInsert().Model(&channelThreadMessage{ChannelID: channelID, Kind: kind, MessageID: messageID, CreatedAt: now}). + On("CONFLICT DO NOTHING"). + Exec(ctx) if err != nil { return false, stack.Wrap(fmt.Errorf("store: claim channel thread message: %w", err)) } diff --git a/acceleration/internal/store/channel_threads_test.go b/acceleration/internal/store/channel_threads_test.go index 57a3bf66b..a3e317ecc 100644 --- a/acceleration/internal/store/channel_threads_test.go +++ b/acceleration/internal/store/channel_threads_test.go @@ -75,6 +75,21 @@ func (s *StoreSuite) TestAnotherThreadGetsAnotherChannel() { s.True(created) } +func (s *StoreSuite) TestTheFirstStarterOfAThreadIsTheOneKept() { + _, err := s.store.LinkChannelThread(s.ctx, thread("thread-one", "C0000CHAN:1")) + s.Require().NoError(err) + starter, err := s.store.ChannelThreadStarter(s.ctx, "thread-one") + s.Require().NoError(err) + s.Empty(starter, "nobody started it yet") + + s.Require().NoError(s.store.SetChannelThreadStarter(s.ctx, "thread-one", "+15550100")) + s.Require().NoError(s.store.SetChannelThreadStarter(s.ctx, "thread-one", "+15550199")) + + starter, err = s.store.ChannelThreadStarter(s.ctx, "thread-one") + s.Require().NoError(err) + s.Equal("+15550100", starter) +} + // One Slack workspace can install two customers' apps, so the same thread is two threads. func (s *StoreSuite) TestTheSameThreadOfAnotherCustomerIsAThreadOfItsOwn() { _, err := s.store.LinkChannelThread(s.ctx, thread("thread-one", "C0000CHAN:1")) diff --git a/acceleration/internal/store/channels.go b/acceleration/internal/store/channels.go index 8d113fde9..ae578e4ed 100644 --- a/acceleration/internal/store/channels.go +++ b/acceleration/internal/store/channels.go @@ -7,6 +7,8 @@ import ( "fmt" "time" + "github.com/uptrace/bun" + "github.com/GetStream/Vision-Agents/acceleration/internal/stack" ) @@ -288,13 +290,21 @@ func (s *Store) ClaimChannelLink(ctx context.Context, customerID, code string) ( // ErrUnknownChannelLink is a code nobody was given, one already spent, or one past its time. var ErrUnknownChannelLink = errors.New("store: there is no such code") +// channelDelivery is one message a line was delivered (ClaimChannelMessage). +type channelDelivery struct { + bun.BaseModel `bun:"table:channel_deliveries,alias:cdel"` + + AccountID string `bun:"account_id,pk"` + MessageID string `bun:"message_id,pk"` + ReceivedAt time.Time `bun:"received_at,notnull"` + FromAddress string `bun:"from_address,notnull"` +} + // ClaimChannelMessage records a message as delivered, and who sent it. False is one // already taken, which a provider retrying a slow delivery sends. func (s *Store) ClaimChannelMessage(ctx context.Context, accountID, messageID, from string) (bool, error) { - result, err := s.db.NewRaw( - "INSERT INTO channel_deliveries (account_id, message_id, received_at, from_address) "+ - "VALUES (?, ?, ?, ?) ON CONFLICT DO NOTHING", - accountID, messageID, time.Now().UTC(), from).Exec(ctx) + delivery := channelDelivery{AccountID: accountID, MessageID: messageID, ReceivedAt: time.Now().UTC(), FromAddress: from} + result, err := s.db.NewInsert().Model(&delivery).On("CONFLICT DO NOTHING").Exec(ctx) if err != nil { return false, fmt.Errorf("store: claim channel message: %w", err) } @@ -309,10 +319,12 @@ func (s *Store) ClaimChannelMessage(ctx context.Context, accountID, messageID, f // they never have. WhatsApp's window for writing first is counted from it. func (s *Store) LastChannelMessageAt(ctx context.Context, accountID, from string) (time.Time, error) { var received time.Time - err := s.db.NewRaw( - "SELECT received_at FROM channel_deliveries "+ - "WHERE account_id = ? AND from_address = ? ORDER BY received_at DESC LIMIT 1", - accountID, from).Scan(ctx, &received) + err := s.db.NewSelect().Model((*channelDelivery)(nil)). + Column("received_at"). + Where("account_id = ? AND from_address = ?", accountID, from). + OrderExpr("received_at DESC"). + Limit(1). + Scan(ctx, &received) if errors.Is(err, sql.ErrNoRows) { return time.Time{}, nil } diff --git a/acceleration/internal/store/chat_channel_turns.go b/acceleration/internal/store/chat_channel_turns.go index f9baf7d42..a5672495c 100644 --- a/acceleration/internal/store/chat_channel_turns.go +++ b/acceleration/internal/store/chat_channel_turns.go @@ -6,9 +6,21 @@ import ( "fmt" "time" + "github.com/uptrace/bun" + "github.com/GetStream/Vision-Agents/acceleration/internal/stack" ) +// chatChannelTurn is the lease on a channel's running turn (TakeChatChannelTurn). +type chatChannelTurn struct { + bun.BaseModel `bun:"table:chat_channel_turns,alias:cht"` + + StreamAppPK int64 `bun:"stream_app_pk,pk"` + CID string `bun:"cid,pk"` + TurnHolder string `bun:"turn_holder,notnull"` + TurnUntil time.Time `bun:"turn_until,notnull"` +} + // TakeChatChannelTurn leases the one running turn of the channel cid of the Stream app app, a // channel of a type an agent config answers in, to holder until until, when no turn holds it // or the one that did ran out. False is a turn another holder has, as TakeChannelThreadTurn @@ -17,12 +29,12 @@ func (s *Store) TakeChatChannelTurn(ctx context.Context, app int64, cid, holder if cid == "" || holder == "" { return false, stack.Wrap(errors.New("store: a channel and a holder are required")) } - result, err := s.db.NewRaw( - "INSERT INTO chat_channel_turns (stream_app_pk, cid, turn_holder, turn_until) VALUES (?, ?, ?, ?) "+ - "ON CONFLICT (stream_app_pk, cid) DO UPDATE "+ - "SET turn_holder = EXCLUDED.turn_holder, turn_until = EXCLUDED.turn_until "+ - "WHERE chat_channel_turns.turn_until < now() OR chat_channel_turns.turn_holder = EXCLUDED.turn_holder", - app, cid, holder, until.UTC()).Exec(ctx) + result, err := s.db.NewInsert().Model(&chatChannelTurn{StreamAppPK: app, CID: cid, TurnHolder: holder, TurnUntil: until.UTC()}). + On("CONFLICT (stream_app_pk, cid) DO UPDATE"). + Set("turn_holder = EXCLUDED.turn_holder"). + Set("turn_until = EXCLUDED.turn_until"). + Where("cht.turn_until < now() OR cht.turn_holder = EXCLUDED.turn_holder"). + Exec(ctx) if err != nil { return false, stack.Wrap(fmt.Errorf("store: take chat channel turn: %w", err)) } @@ -36,7 +48,8 @@ func (s *Store) TakeChatChannelTurn(ctx context.Context, app int64, cid, holder // ReleaseChatChannelTurn ends holder's lease on a channel's turn, so the next turn need not // wait for it to run out. A lease another holder took since is left alone. func (s *Store) ReleaseChatChannelTurn(ctx context.Context, app int64, cid, holder string) error { - _, err := s.db.NewRaw("DELETE FROM chat_channel_turns WHERE stream_app_pk = ? AND cid = ? AND turn_holder = ?", - app, cid, holder).Exec(ctx) + _, err := s.db.NewDelete().Model((*chatChannelTurn)(nil)). + Where("stream_app_pk = ? AND cid = ? AND turn_holder = ?", app, cid, holder). + Exec(ctx) return stack.Wrap(err) } diff --git a/acceleration/internal/store/connection_events.go b/acceleration/internal/store/connection_events.go index 56d2963af..097cfe8dc 100644 --- a/acceleration/internal/store/connection_events.go +++ b/acceleration/internal/store/connection_events.go @@ -204,7 +204,7 @@ func (s *Store) claimConnectionEventSubscriptions(ctx context.Context, query str // found is false when none is ever due: there are no subscriptions, or none expires. func (s *Store) NextConnectionEventSubscriptionAt(ctx context.Context) (next time.Time, found bool, err error) { var at sql.NullTime - if err := s.db.NewRaw("SELECT min(next_attempt_at) FROM connection_event_subscriptions").Scan(ctx, &at); err != nil { + if err := s.db.NewSelect().Model((*ConnectionEventSubscription)(nil)).ColumnExpr("min(next_attempt_at)").Scan(ctx, &at); err != nil { return time.Time{}, false, stack.Wrap(fmt.Errorf("store: next connection event subscription: %w", err)) } return at.Time, at.Valid, nil @@ -245,12 +245,20 @@ func (s *Store) DeleteConnectionEventSubscriptions(ctx context.Context, customer return nil } +// connectionEventDelivery is one event a subscription got (ClaimConnectionEvent). +type connectionEventDelivery struct { + bun.BaseModel `bun:"table:connection_event_deliveries,alias:cevd"` + + SubscriptionID string `bun:"subscription_id,pk"` + EventID string `bun:"event_id,pk"` + ReceivedAt time.Time `bun:"received_at,notnull"` +} + // ClaimConnectionEvent records that a subscription's event arrived, and reports whether this // is the first time: a retried delivery of the same event id is not. func (s *Store) ClaimConnectionEvent(ctx context.Context, subscriptionID, eventID string) (bool, error) { - result, err := s.db.NewRaw( - "INSERT INTO connection_event_deliveries (subscription_id, event_id, received_at) VALUES (?, ?, ?) ON CONFLICT DO NOTHING", - subscriptionID, eventID, time.Now().UTC()).Exec(ctx) + delivery := connectionEventDelivery{SubscriptionID: subscriptionID, EventID: eventID, ReceivedAt: time.Now().UTC()} + result, err := s.db.NewInsert().Model(&delivery).On("CONFLICT DO NOTHING").Exec(ctx) if err != nil { return false, stack.Wrap(fmt.Errorf("store: claim connection event: %w", err)) } diff --git a/acceleration/internal/store/grants.go b/acceleration/internal/store/grants.go index 03e5ee4de..d676d2a83 100644 --- a/acceleration/internal/store/grants.go +++ b/acceleration/internal/store/grants.go @@ -452,6 +452,24 @@ DELETE FROM limit_grant_uses WHERE (grant_id, usage_id) IN ( } } +// grantUse is one row counted toward a grant (limit_grant_uses). +type grantUse struct { + bun.BaseModel `bun:"table:limit_grant_uses,alias:lgu"` + + GrantID int64 `bun:"grant_id,pk"` + UsageID uuid.UUID `bun:"usage_id,pk,type:uuid"` + Micros int64 `bun:"micros,notnull"` +} + +// grantReconcile is where a shard's reconcile of grant uses got (limit_grant_reconciles). +type grantReconcile struct { + bun.BaseModel `bun:"table:limit_grant_reconciles,alias:lgr"` + + Shard string `bun:"shard,pk"` + StartedAt time.Time `bun:"started_at,notnull"` + UpdatedAt time.Time `bun:"updated_at,notnull"` +} + // GrantUsesCounted is, of the rows named by usage id, how many grants each counts toward. func (s *Store) GrantUsesCounted(ctx context.Context, usageIDs []uuid.UUID) (map[uuid.UUID]int, error) { counted := map[uuid.UUID]int{} @@ -466,8 +484,12 @@ func (s *Store) GrantUsesCounted(ctx context.Context, usageIDs []uuid.UUID) (map UsageID uuid.UUID `bun:"usage_id"` Grants int `bun:"grants"` } - err := s.db.NewRaw("SELECT usage_id, count(*) AS grants FROM limit_grant_uses WHERE usage_id = ANY(?::uuid[]) GROUP BY usage_id", - pgdialect.Array(ids)).Scan(ctx, &rows) + err := s.db.NewSelect().Model((*grantUse)(nil)). + Column("usage_id"). + ColumnExpr("count(*) AS grants"). + Where("usage_id = ANY(?::uuid[])", pgdialect.Array(ids)). + Group("usage_id"). + Scan(ctx, &rows) if err != nil { return nil, stack.Wrap(fmt.Errorf("store: grant uses counted: %w", err)) } @@ -541,7 +563,7 @@ const reconcileGrantsQuery = "SELECT id, started_at, usage_id, cost_micros, limi // reconcileShard is ReconcileGrantUses for one shard, read up to now. func (s *Store) reconcileShard(ctx context.Context, shard bun.IDB, name string, now time.Time) (int, error) { var mark time.Time - err := s.db.NewRaw("SELECT started_at FROM limit_grant_reconciles WHERE shard = ?", name).Scan(ctx, &mark) + err := s.db.NewSelect().Model((*grantReconcile)(nil)).Column("started_at").Where("shard = ?", name).Scan(ctx, &mark) if err != nil && !errors.Is(err, sql.ErrNoRows) { return 0, fmt.Errorf("shard %s: %w", name, err) } @@ -582,9 +604,13 @@ func (s *Store) reconcileShard(ctx context.Context, shard bun.IDB, name string, reached = last.StartedAt } } - if _, err := s.db.NewRaw("INSERT INTO limit_grant_reconciles (shard, started_at) VALUES (?, ?) "+ - "ON CONFLICT (shard) DO UPDATE SET started_at = GREATEST(limit_grant_reconciles.started_at, EXCLUDED.started_at), updated_at = now()", - name, reached).Exec(ctx); err != nil { + _, err = s.db.NewInsert().Model(&grantReconcile{Shard: name, StartedAt: reached}). + Column("shard", "started_at"). + On("CONFLICT (shard) DO UPDATE"). + Set("started_at = GREATEST(lgr.started_at, EXCLUDED.started_at)"). + Set("updated_at = now()"). + Exec(ctx) + if err != nil { return counted, fmt.Errorf("shard %s: keep where it got: %w", name, err) } if len(rows) < grantReconcilePage { diff --git a/acceleration/internal/store/grants_test.go b/acceleration/internal/store/grants_test.go index b6c4e50ad..989b576ca 100644 --- a/acceleration/internal/store/grants_test.go +++ b/acceleration/internal/store/grants_test.go @@ -273,6 +273,22 @@ func (s *StoreSuite) TestARowWrittenTwiceCountsOnceTowardItsGrant() { s.EqualValues(350, s.used(grant.ID), "a row sent twice in one batch counts once") } +func (s *StoreSuite) TestGrantUsesCountedIsHowManyGrantsEachRowCountsToward() { + a, b := s.grant(LimitGrant{AmountMicros: 10_000}), s.grant(LimitGrant{AmountMicros: 10_000}) + both, one, other, unasked := uuid.Must(uuid.NewV7()), uuid.Must(uuid.NewV7()), uuid.Must(uuid.NewV7()), uuid.Must(uuid.NewV7()) + _, err := s.store.AddGrantUses(s.ctx, []GrantUse{ + {GrantID: a.ID, UsageID: both, Micros: 10}, + {GrantID: b.ID, UsageID: both, Micros: 10}, + {GrantID: a.ID, UsageID: one, Micros: 10}, + {GrantID: b.ID, UsageID: unasked, Micros: 10}, + }) + s.Require().NoError(err) + + counted, err := s.store.GrantUsesCounted(s.ctx, []uuid.UUID{both, one, other}) + s.Require().NoError(err) + s.Equal(map[uuid.UUID]int{both: 2, one: 1}, counted, "a row counted toward nothing, or not asked about, is left out") +} + func (s *StoreSuite) TestAUsedUpGrantIsNotLiveAndLeavesRoomForAnother() { for range MaxLiveGrants - 1 { s.grant(LimitGrant{OrganizationID: "org-used"}) diff --git a/acceleration/internal/store/hooks.go b/acceleration/internal/store/hooks.go index aa0df7735..b73f578d0 100644 --- a/acceleration/internal/store/hooks.go +++ b/acceleration/internal/store/hooks.go @@ -155,13 +155,22 @@ func (s *Store) FirstDelivery(ctx context.Context, key string) (bool, error) { return recordDelivery(ctx, s.db, key) } +// hookDelivery is one hook delivery's key, as FirstDelivery records it. +type hookDelivery struct { + bun.BaseModel `bun:"table:hook_deliveries,alias:hd"` + + Key string `bun:"key,pk"` + SeenAt time.Time `bun:"seen_at,notnull"` +} + // forgetDeliveries deletes the deliveries older than a day, at most once every // deliveriesPrunedEvery. func (s *Store) forgetDeliveries(ctx context.Context) error { now := time.Now().UTC() if last := s.deliveriesPruned.Load(); now.UnixNano()-last >= int64(deliveriesPrunedEvery) && s.deliveriesPruned.CompareAndSwap(last, now.UnixNano()) { - if _, err := s.db.ExecContext(ctx, "DELETE FROM hook_deliveries WHERE seen_at < ?", now.Add(-deliveryKeptFor)); err != nil { + _, err := s.db.NewDelete().Model((*hookDelivery)(nil)).Where("seen_at < ?", now.Add(-deliveryKeptFor)).Exec(ctx) + if err != nil { return fmt.Errorf("store: forget hook deliveries: %w", err) } } @@ -170,8 +179,10 @@ func (s *Store) forgetDeliveries(ctx context.Context) error { // recordDelivery is FirstDelivery's record on db, the store's or a transaction's. func recordDelivery(ctx context.Context, db bun.IDB, key string) (bool, error) { - result, err := db.NewRaw( - "INSERT INTO hook_deliveries (key, seen_at) VALUES (?, now()) ON CONFLICT (key) DO NOTHING", key).Exec(ctx) + result, err := db.NewInsert().Model(&hookDelivery{Key: key}). + Column("key"). + On("CONFLICT (key) DO NOTHING"). + Exec(ctx) if err != nil { return false, fmt.Errorf("store: record hook delivery: %w", err) } diff --git a/acceleration/internal/store/movecursor.go b/acceleration/internal/store/movecursor.go index 53e6a75c6..9b65cf3f2 100644 --- a/acceleration/internal/store/movecursor.go +++ b/acceleration/internal/store/movecursor.go @@ -6,6 +6,8 @@ import ( "errors" "fmt" "time" + + "github.com/uptrace/bun" ) // MoveCursor is where a copy of an app from another deployment stands here. @@ -17,31 +19,39 @@ type MoveCursor struct { ReadAt time.Time } +// dataMoveCursor is MoveCursor as data_move_cursors holds it. +type dataMoveCursor struct { + bun.BaseModel `bun:"table:data_move_cursors,alias:dmc"` + + CustomerID string `bun:"customer_id,pk"` + Source string `bun:"source,pk"` + Cursor *int64 `bun:"cursor"` + ReadAt time.Time `bun:"read_at,notnull"` +} + // MoveCursor answers where the copy of a customer from a source stands here, nil when none was // ever started. func (s *Store) MoveCursor(ctx context.Context, customerID, source string) (*MoveCursor, error) { - var saved MoveCursor - err := s.db.QueryRowContext(ctx, "SELECT cursor, read_at FROM data_move_cursors WHERE customer_id = ? AND source = ?", - customerID, source).Scan(&saved.Cursor, &saved.ReadAt) + var saved dataMoveCursor + err := s.db.NewSelect().Model(&saved).Where("customer_id = ? AND source = ?", customerID, source).Scan(ctx) if errors.Is(err, sql.ErrNoRows) { return nil, nil } if err != nil { return nil, fmt.Errorf("store: move cursor: %w", err) } - return &saved, nil + return &MoveCursor{Cursor: saved.Cursor, ReadAt: saved.ReadAt}, nil } // SaveMoveCursor records where the copy of a customer from a source stands here: the cursor the // source was read up to at readAt, or nil for a copy that has started and not finished. A cursor // never moves back: of two runs at once, the one that read less cannot undo the other. func (s *Store) SaveMoveCursor(ctx context.Context, customerID, source string, cursor *int64, readAt time.Time) error { - _, err := s.db.ExecContext(ctx, ` -INSERT INTO data_move_cursors (customer_id, source, cursor, read_at) VALUES (?, ?, ?, ?) -ON CONFLICT (customer_id, source) DO UPDATE SET - cursor = CASE WHEN EXCLUDED.cursor IS NULL THEN NULL ELSE GREATEST(data_move_cursors.cursor, EXCLUDED.cursor) END, - read_at = CASE WHEN EXCLUDED.cursor IS NULL THEN EXCLUDED.read_at ELSE GREATEST(data_move_cursors.read_at, EXCLUDED.read_at) END`, - customerID, source, cursor, readAt) + _, err := s.db.NewInsert().Model(&dataMoveCursor{CustomerID: customerID, Source: source, Cursor: cursor, ReadAt: readAt}). + On("CONFLICT (customer_id, source) DO UPDATE"). + Set("cursor = CASE WHEN EXCLUDED.cursor IS NULL THEN NULL ELSE GREATEST(dmc.cursor, EXCLUDED.cursor) END"). + Set("read_at = CASE WHEN EXCLUDED.cursor IS NULL THEN EXCLUDED.read_at ELSE GREATEST(dmc.read_at, EXCLUDED.read_at) END"). + Exec(ctx) if err != nil { return fmt.Errorf("store: save move cursor: %w", err) }