diff --git a/acceleration/internal/store/balance_notices.go b/acceleration/internal/store/balance_notices.go index 77e0a8c26..f61c09dbd 100644 --- a/acceleration/internal/store/balance_notices.go +++ b/acceleration/internal/store/balance_notices.go @@ -2,7 +2,6 @@ package store import ( "context" - "encoding/json" "errors" "fmt" "time" @@ -49,10 +48,6 @@ func (s *Store) NoteBalance(ctx context.Context, notice BalanceNotice, delivery if notice.Kind != NoticeBalanceLow && notice.Kind != NoticeBalanceExhausted { return false, 0, stack.Wrap(fmt.Errorf("store: a balance notice is %s or %s, not %q", NoticeBalanceLow, NoticeBalanceExhausted, notice.Kind)) } - headers, err := json.Marshal(delivery.Headers) - if err != nil { - return false, 0, stack.Wrap(err) - } err = s.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { written, err := tx.NewInsert().Model(¬ice).On("CONFLICT DO NOTHING").Exec(ctx) if err != nil { @@ -80,11 +75,15 @@ func (s *Store) NoteBalance(ctx context.Context, notice BalanceNotice, delivery if err != nil { return err } - if _, err := tx.NewRaw(` -INSERT INTO connector_event_deliveries (destination_id, id, headers, body, attempts, next_attempt_at) -VALUES (?, ?, ?::jsonb, ?, 0, ?) -ON CONFLICT (destination_id, id) DO NOTHING`, - destination.ID, delivery.ID, string(headers), payload, delivery.NextAttemptAt.UTC()).Exec(ctx); err != nil { + row := EventDelivery{ + DestinationID: destination.ID, ID: delivery.ID, Headers: delivery.Headers, Body: payload, + NextAttemptAt: delivery.NextAttemptAt.UTC(), + } + _, err = tx.NewInsert().Model(&row). + Column("destination_id", "id", "headers", "body", "attempts", "next_attempt_at"). + On("CONFLICT (destination_id, id) DO NOTHING"). + Exec(ctx) + if err != nil { return err } queued++ diff --git a/acceleration/internal/store/campaign_analytics.go b/acceleration/internal/store/campaign_analytics.go index 20b08ffb9..7ae6306ab 100644 --- a/acceleration/internal/store/campaign_analytics.go +++ b/acceleration/internal/store/campaign_analytics.go @@ -72,8 +72,12 @@ WHERE cc.campaign_id = ? AND EXISTS (SELECT 1 FROM campaign_attempts ca Outcome string Count int } - err = s.db.NewRaw(`SELECT outcome, count(*) AS count FROM campaign_contacts -WHERE campaign_id = ? AND stage = ? GROUP BY outcome`, campaignID, ContactCompleted).Scan(ctx, &outcomes) + err = s.db.NewSelect().Model((*Contact)(nil)). + Column("cc.outcome"). + ColumnExpr("count(*) AS count"). + Where("cc.campaign_id = ? AND cc.stage = ?", campaignID, ContactCompleted). + Group("cc.outcome"). + Scan(ctx, &outcomes) if err != nil { return CampaignAnalytics{}, stack.Wrap(fmt.Errorf("store: count outcomes: %w", err)) } @@ -85,8 +89,12 @@ WHERE campaign_id = ? AND stage = ? GROUP BY outcome`, campaignID, ContactComple CallOutcome string Count int } - err = s.db.NewRaw(`SELECT call_outcome, count(*) AS count FROM campaign_attempts -WHERE campaign_id = ? AND call_outcome IS NOT NULL GROUP BY call_outcome`, campaignID).Scan(ctx, &calls) + err = s.db.NewSelect().Model((*CampaignAttempt)(nil)). + Column("ca.call_outcome"). + ColumnExpr("count(*) AS count"). + Where("ca.campaign_id = ? AND ca.call_outcome IS NOT NULL", campaignID). + Group("ca.call_outcome"). + Scan(ctx, &calls) if err != nil { return CampaignAnalytics{}, stack.Wrap(fmt.Errorf("store: count call outcomes: %w", err)) } @@ -119,9 +127,13 @@ func (s *Store) ContactStages(ctx context.Context, campaignIDs []string) (map[st Stage string Count int } - err := s.db.NewRaw(`SELECT campaign_id, CASE WHEN stage = ? THEN ? ELSE stage END AS stage, count(*) AS count -FROM campaign_contacts WHERE campaign_id IN (?) GROUP BY 1, 2`, - AwaitingOutcome, InProgress, bun.In(campaignIDs)).Scan(ctx, &rows) + err := s.db.NewSelect().Model((*Contact)(nil)). + Column("cc.campaign_id"). + ColumnExpr("CASE WHEN cc.stage = ? THEN ? ELSE cc.stage END AS stage", AwaitingOutcome, InProgress). + ColumnExpr("count(*) AS count"). + Where("cc.campaign_id IN (?)", bun.In(campaignIDs)). + GroupExpr("1, 2"). + Scan(ctx, &rows) if err != nil { return nil, stack.Wrap(fmt.Errorf("store: count contact stages: %w", err)) } diff --git a/acceleration/internal/store/campaign_dispatch.go b/acceleration/internal/store/campaign_dispatch.go index f89383d1f..e777da587 100644 --- a/acceleration/internal/store/campaign_dispatch.go +++ b/acceleration/internal/store/campaign_dispatch.go @@ -140,18 +140,21 @@ func (s *Store) TakeOverExpired(ctx context.Context, owner string, now time.Time return nil, nil } + expired := s.db.NewSelect().Model((*Contact)(nil)). + Column("cc.id"). + Where("cc.stage = ? AND cc.leased_until < ?", InProgress, now). + Where("NOT EXISTS (SELECT 1 FROM campaigns cp WHERE cp.id = cc.campaign_id AND cp.move_paused)"). + Order("cc.leased_until"). + Limit(limit). + For("UPDATE SKIP LOCKED") var contacts []Contact - err := s.db.NewRaw(`WITH expired AS ( - SELECT id FROM campaign_contacts cc - WHERE stage = ? AND leased_until < ? - AND NOT EXISTS (SELECT 1 FROM campaigns cp WHERE cp.id = cc.campaign_id AND cp.move_paused) - ORDER BY leased_until - LIMIT ? - FOR UPDATE SKIP LOCKED -) -UPDATE campaign_contacts cc SET leased_by = ?, leased_until = ? -FROM expired WHERE cc.id = expired.id -RETURNING cc.*`, InProgress, now, limit, owner, now.Add(lease)).Scan(ctx, &contacts) + err := s.db.NewUpdate().With("expired", expired).Model((*Contact)(nil)). + TableExpr("expired"). + Set("leased_by = ?", owner). + Set("leased_until = ?", now.Add(lease)). + Where("cc.id = expired.id"). + Returning("cc.*"). + Scan(ctx, &contacts) if err != nil { return nil, stack.Wrap(fmt.Errorf("store: take over expired leases: %w", err)) } @@ -457,10 +460,13 @@ func (s *Store) PutOffClassification(ctx context.Context, owner, attemptID strin } var released int64 err := s.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { - result, err := tx.ExecContext(ctx, `UPDATE campaign_contacts cc SET leased_by = '', leased_until = NULL -FROM campaign_attempts ca -WHERE ca.id = ? AND ca.classify_state = ? AND cc.id = ca.contact_id AND cc.stage = ? AND cc.leased_by = ?`, - attemptID, classifyPending, AwaitingOutcome, owner) + result, err := tx.NewUpdate().Model((*Contact)(nil)). + TableExpr("campaign_attempts AS ca"). + Set("leased_by = ''"). + Set("leased_until = NULL"). + Where("ca.id = ? AND ca.classify_state = ?", attemptID, classifyPending). + Where("cc.id = ca.contact_id AND cc.stage = ? AND cc.leased_by = ?", AwaitingOutcome, owner). + Exec(ctx) if err != nil { return fmt.Errorf("store: release a contact to classify later: %w", err) } diff --git a/acceleration/internal/store/channel_inbound.go b/acceleration/internal/store/channel_inbound.go index ac9973789..136d9163d 100644 --- a/acceleration/internal/store/channel_inbound.go +++ b/acceleration/internal/store/channel_inbound.go @@ -215,6 +215,8 @@ func queueChannelInbound(ctx context.Context, tx bun.Tx, message ChannelInbound) if _, err := tx.NewRaw("SELECT 1 FROM channel_threads WHERE channel_id = ? FOR NO KEY UPDATE", message.ChannelID).Exec(ctx); err != nil { return false, stack.Wrap(fmt.Errorf("store: queue channel inbound: %w", err)) } + // Not the insert builder: bun writes a JSON \u0000 as the text \\u0000, and a delivery jsonb + // cannot hold must fail here, so it stays unclaimed and the provider's retry takes it again. media, err := json.Marshal(message.Media) if err != nil { return false, stack.Wrap(err) @@ -309,13 +311,13 @@ func (s *Store) claimChannelInbound(ctx context.Context, query string, now time. // found is false when no message is queued at all. func (s *Store) NextChannelInboundAt(ctx context.Context) (next time.Time, found bool, err error) { var at sql.NullTime - if err := s.db.NewRaw(` -SELECT min(head.next_attempt_at) FROM ( - SELECT DISTINCT ON (channel_id) next_attempt_at - FROM channel_thread_inbound - WHERE state = 'queued' - ORDER BY channel_id, seq -) AS head`).Scan(ctx, &at); err != nil { + head := s.db.NewSelect().Model((*ChannelInbound)(nil)). + DistinctOn("cti.channel_id"). + Column("cti.next_attempt_at"). + Where("cti.state = ?", ChannelInboundQueued). + Order("cti.channel_id", "cti.seq") + err = s.db.NewSelect().TableExpr("(?) AS head", head).ColumnExpr("min(head.next_attempt_at)").Scan(ctx, &at) + if err != nil { return time.Time{}, false, stack.Wrap(fmt.Errorf("store: next channel inbound: %w", err)) } return at.Time, at.Valid, nil @@ -357,8 +359,11 @@ func (s *Store) FailChannelInbound(ctx context.Context, claimed ChannelInbound, if forward(ctx) != nil { return nil } - if _, err := tx.NewRaw("UPDATE channel_thread_inbound SET forward = jsonb_set(forward, '{forwarded}', 'true') WHERE seq = ?", - claimed.Seq).Exec(ctx); err != nil { + _, err = tx.NewUpdate().Model((*ChannelInbound)(nil)). + Set("forward = jsonb_set(forward, '{forwarded}', 'true')"). + Where("seq = ?", claimed.Seq). + Exec(ctx) + if err != nil { return fmt.Errorf("store: fail channel inbound: %w", err) } return nil @@ -383,13 +388,14 @@ func lockChannelInboundDelivery(ctx context.Context, tx bun.Tx, claimed ChannelI // it: one with the same provider id for the delivery, or with none and the same body, as the // event destinations key a forward (eventforward.deliveryID). func channelInboundDeliveryForwarded(ctx context.Context, tx bun.Tx, claimed ChannelInbound) (bool, error) { - var forwarded bool - if err := tx.NewRaw(` -SELECT EXISTS ( - SELECT 1 FROM channel_thread_inbound - WHERE state = 'failed' AND forward->>'forwarded' = 'true' AND customer_id = ? AND connector_id = ? - AND forward->>'id' = ? AND (? <> '' OR raw = ?) -)`, claimed.CustomerID, claimed.ConnectorID, claimed.Forward.ID, claimed.Forward.ID, claimed.Raw).Scan(ctx, &forwarded); err != nil { + forwarded, err := tx.NewSelect().Model((*ChannelInbound)(nil)). + Where("state = ?", ChannelInboundFailed). + Where("forward->>'forwarded' = 'true'"). + Where("customer_id = ? AND connector_id = ?", claimed.CustomerID, claimed.ConnectorID). + Where("forward->>'id' = ?", claimed.Forward.ID). + Where("(? <> '' OR raw = ?)", claimed.Forward.ID, claimed.Raw). + Exists(ctx) + if err != nil { return false, fmt.Errorf("store: fail channel inbound: %w", err) } return forwarded, nil @@ -452,10 +458,10 @@ func updateClaimedChannelInbound(ctx context.Context, claimed ChannelInbound, up // PruneChannelInbound deletes the rows of written messages finished a day before now, and of // given-up ones 30 days before, and returns how many it deleted. func (s *Store) PruneChannelInbound(ctx context.Context, now time.Time) (int, error) { - result, err := s.db.NewRaw(` -DELETE FROM channel_thread_inbound -WHERE (state = 'done' AND finished_at < ?) OR (state = 'failed' AND finished_at < ?)`, - now.Add(-channelInboundDoneKeep).UTC(), now.Add(-channelInboundFailedKeep).UTC()).Exec(ctx) + result, err := s.db.NewDelete().Model((*ChannelInbound)(nil)). + Where("(state = ? AND finished_at < ?) OR (state = ? AND finished_at < ?)", + ChannelInboundDone, now.Add(-channelInboundDoneKeep).UTC(), ChannelInboundFailed, now.Add(-channelInboundFailedKeep).UTC()). + Exec(ctx) if err != nil { return 0, stack.Wrap(fmt.Errorf("store: prune channel inbound: %w", err)) } diff --git a/acceleration/internal/store/chat_inbound.go b/acceleration/internal/store/chat_inbound.go index 36e1ab824..142e264bc 100644 --- a/acceleration/internal/store/chat_inbound.go +++ b/acceleration/internal/store/chat_inbound.go @@ -82,12 +82,11 @@ func queueChatInbound(ctx context.Context, db bun.IDB, delivery ChatInbound) (bo if delivery.StreamApp == 0 { return false, stack.Wrap(errors.New("store: a Stream app is required: a message queued without one would be queued again once the app is known")) } - result, err := db.NewRaw(` -INSERT INTO chat_inbound (stream_app, message_id, customer_id, deployment, api_key, body, next_attempt_at) -VALUES (?, ?, ?, ?, ?, ?, ?) -ON CONFLICT (stream_app, message_id) DO NOTHING`, - delivery.StreamApp, delivery.MessageID, delivery.CustomerID, delivery.Deployment, delivery.APIKey, delivery.Body, - delivery.NextAttemptAt.UTC()).Exec(ctx) + delivery.NextAttemptAt = delivery.NextAttemptAt.UTC() + result, err := db.NewInsert().Model(&delivery). + Column("stream_app", "message_id", "customer_id", "deployment", "api_key", "body", "next_attempt_at"). + On("CONFLICT (stream_app, message_id) DO NOTHING"). + Exec(ctx) if err != nil { return false, stack.Wrap(fmt.Errorf("store: queue chat inbound: %w", err)) } @@ -164,7 +163,7 @@ func (s *Store) claimChatInbound(ctx context.Context, query string, now time.Tim // ChatInboundBySeq returns the message queued as seq, in whatever state it is. func (s *Store) ChatInboundBySeq(ctx context.Context, seq int64) (ChatInbound, error) { var delivery ChatInbound - err := s.db.NewRaw("SELECT "+chatInboundColumns+" FROM chat_inbound AS cd WHERE cd.seq = ?", seq).Scan(ctx, &delivery) + err := s.db.NewSelect().Model(&delivery).Where("cd.seq = ?", seq).Scan(ctx) if errors.Is(err, sql.ErrNoRows) { return ChatInbound{}, stack.Wrap(fmt.Errorf("%w: %d", ErrNoChatInbound, seq)) } @@ -178,7 +177,11 @@ func (s *Store) ChatInboundBySeq(ctx context.Context, seq int64) (ChatInbound, e // their lease. found is false when no message is queued at all. func (s *Store) NextChatInboundAt(ctx context.Context) (next time.Time, found bool, err error) { var at sql.NullTime - if err := s.db.NewRaw("SELECT min(next_attempt_at) FROM chat_inbound WHERE state = 'queued'").Scan(ctx, &at); err != nil { + err = s.db.NewSelect().Model((*ChatInbound)(nil)). + ColumnExpr("min(next_attempt_at)"). + Where("state = ?", ChatInboundQueued). + Scan(ctx, &at) + if err != nil { return time.Time{}, false, stack.Wrap(fmt.Errorf("store: next chat inbound: %w", err)) } return at.Time, at.Valid, nil @@ -239,10 +242,12 @@ func updateClaimedChatInbound(ctx context.Context, claimed ChatInbound, states [ // nothing is queued under streamApp and messageID: there is nothing to answer then. The lease // is left as it is, for the worker holding it to finish the row (FinishChatInbound). func (s *Store) StartChatInboundAnswer(ctx context.Context, streamApp int64, messageID string, at time.Time) (started bool, err error) { - result, err := s.db.NewRaw(` -UPDATE chat_inbound SET state = 'done', finished_at = ? -WHERE stream_app = ? AND message_id = ? AND state = 'queued'`, - at.UTC(), streamApp, messageID).Exec(ctx) + result, err := s.db.NewUpdate().Model((*ChatInbound)(nil)). + Set("state = ?", ChatInboundDone). + Set("finished_at = ?", at.UTC()). + Where("stream_app = ? AND message_id = ?", streamApp, messageID). + Where("state = ?", ChatInboundQueued). + Exec(ctx) if err != nil { return false, stack.Wrap(fmt.Errorf("store: start chat inbound answer: %w", err)) } @@ -257,10 +262,12 @@ WHERE stream_app = ? AND message_id = ? AND state = 'queued'`, // messageID said, so a later try of it hands that on rather than hearing it again. What was // kept first stays. func (s *Store) KeepChatInboundHeard(ctx context.Context, streamApp int64, messageID, heard string) error { - if _, err := s.db.NewRaw(` -UPDATE chat_inbound SET heard = ? -WHERE stream_app = ? AND message_id = ? AND heard IS NULL`, - heard, streamApp, messageID).Exec(ctx); err != nil { + _, err := s.db.NewUpdate().Model((*ChatInbound)(nil)). + Set("heard = ?", heard). + Where("stream_app = ? AND message_id = ?", streamApp, messageID). + Where("heard IS NULL"). + Exec(ctx) + if err != nil { return stack.Wrap(fmt.Errorf("store: keep chat inbound heard: %w", err)) } return nil @@ -270,10 +277,13 @@ WHERE stream_app = ? AND message_id = ? AND heard IS NULL`, // and that nothing then took, such as a dispatch worker that was not connected, so a later try // answers it. The lease is left as it is, for the worker holding it to retry the row. func (s *Store) ReleaseChatInboundAnswer(ctx context.Context, streamApp int64, messageID string) error { - if _, err := s.db.NewRaw(` -UPDATE chat_inbound SET state = 'queued', finished_at = NULL -WHERE stream_app = ? AND message_id = ? AND state = 'done'`, - streamApp, messageID).Exec(ctx); err != nil { + _, err := s.db.NewUpdate().Model((*ChatInbound)(nil)). + Set("state = ?", ChatInboundQueued). + Set("finished_at = NULL"). + Where("stream_app = ? AND message_id = ?", streamApp, messageID). + Where("state = ?", ChatInboundDone). + Exec(ctx) + if err != nil { return stack.Wrap(fmt.Errorf("store: release chat inbound answer: %w", err)) } return nil @@ -282,10 +292,10 @@ WHERE stream_app = ? AND message_id = ? AND state = 'done'`, // PruneChatInbound deletes the rows of answered messages finished a day before now, and of // given-up ones 30 days before, and returns how many it deleted. func (s *Store) PruneChatInbound(ctx context.Context, now time.Time) (int, error) { - result, err := s.db.NewRaw(` -DELETE FROM chat_inbound -WHERE (state = 'done' AND finished_at < ?) OR (state = 'failed' AND finished_at < ?)`, - now.Add(-chatInboundDoneKeep).UTC(), now.Add(-chatInboundFailedKeep).UTC()).Exec(ctx) + result, err := s.db.NewDelete().Model((*ChatInbound)(nil)). + Where("(state = ? AND finished_at < ?) OR (state = ? AND finished_at < ?)", + ChatInboundDone, now.Add(-chatInboundDoneKeep).UTC(), ChatInboundFailed, now.Add(-chatInboundFailedKeep).UTC()). + Exec(ctx) if err != nil { return 0, stack.Wrap(fmt.Errorf("store: prune chat inbound: %w", err)) } diff --git a/acceleration/internal/store/config_tokens.go b/acceleration/internal/store/config_tokens.go index 80c0f16e8..974c39da5 100644 --- a/acceleration/internal/store/config_tokens.go +++ b/acceleration/internal/store/config_tokens.go @@ -7,6 +7,8 @@ import ( "fmt" "time" + "github.com/uptrace/bun" + "github.com/GetStream/Vision-Agents/acceleration/internal/stack" ) @@ -23,6 +25,8 @@ var ErrNoConnectorConfigToken = errors.New("store: no configuration token for th // router for one connector, with its refresh token, sealed by the caller as one blob. It is // what creates, updates and deletes the customer's managed provider app. type ConnectorConfigToken struct { + bun.BaseModel `bun:"table:connector_config_tokens,alias:cct"` + CustomerID string `bun:"customer_id,pk"` ConnectorID string `bun:"connector_id,pk"` TokensSealed []byte `bun:"tokens_sealed,notnull"` @@ -62,21 +66,20 @@ func (s *Store) PutConnectorConfigToken(ctx context.Context, token *ConnectorCon ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), credentialDetachedTimeout) defer cancel() now := time.Now().UTC().Truncate(time.Microsecond) - err := s.db.NewRaw(` -INSERT INTO connector_config_tokens AS cct - (customer_id, connector_id, tokens_sealed, kek_version, expires_at, created_at, updated_at) -VALUES (?, ?, ?, ?, ?, ?, ?) -ON CONFLICT (customer_id, connector_id) DO UPDATE -SET tokens_sealed = EXCLUDED.tokens_sealed, - kek_version = EXCLUDED.kek_version, - expires_at = EXCLUDED.expires_at, - updated_at = EXCLUDED.updated_at -RETURNING cct.created_at`, - token.CustomerID, token.ConnectorID, token.TokensSealed, token.KEKVersion, token.ExpiresAt.UTC(), now, now). - Scan(ctx, &token.CreatedAt) + row := *token + row.CreatedAt, row.UpdatedAt = now, now + err := s.db.NewInsert().Model(&row). + On("CONFLICT (customer_id, connector_id) DO UPDATE"). + Set("tokens_sealed = EXCLUDED.tokens_sealed"). + Set("kek_version = EXCLUDED.kek_version"). + Set("expires_at = EXCLUDED.expires_at"). + Set("updated_at = EXCLUDED.updated_at"). + Returning("cct.created_at"). + Scan(ctx) if err != nil { return stack.Wrap(fmt.Errorf("store: put connector config token: %w", err)) } + token.CreatedAt = row.CreatedAt token.UpdatedAt = now return nil } @@ -87,10 +90,9 @@ func (s *Store) ConnectorConfigToken(ctx context.Context, customerID, connectorI return ConnectorConfigToken{}, stack.Wrap(errors.New("store: a customer and a connector id are required")) } var token ConnectorConfigToken - err := s.db.NewRaw(` -SELECT customer_id, connector_id, tokens_sealed, kek_version, expires_at, created_at, updated_at -FROM connector_config_tokens WHERE customer_id = ? AND connector_id = ?`, customerID, connectorID). - Scan(ctx, &token.CustomerID, &token.ConnectorID, &token.TokensSealed, &token.KEKVersion, &token.ExpiresAt, &token.CreatedAt, &token.UpdatedAt) + err := s.db.NewSelect().Model(&token). + Where("cct.customer_id = ? AND cct.connector_id = ?", customerID, connectorID). + Scan(ctx) if errors.Is(err, sql.ErrNoRows) { return ConnectorConfigToken{}, stack.Wrap(fmt.Errorf("%w: %s", ErrNoConnectorConfigToken, connectorID)) } @@ -106,7 +108,9 @@ func (s *Store) DeleteConnectorConfigToken(ctx context.Context, customerID, conn if customerID == "" || connectorID == "" { return stack.Wrap(errors.New("store: a customer and a connector id are required")) } - _, err := s.db.ExecContext(ctx, "DELETE FROM connector_config_tokens WHERE customer_id = ? AND connector_id = ?", customerID, connectorID) + _, err := s.db.NewDelete().Model((*ConnectorConfigToken)(nil)). + Where("customer_id = ? AND connector_id = ?", customerID, connectorID). + Exec(ctx) if err != nil { return stack.Wrap(fmt.Errorf("store: delete connector config token: %w", err)) } diff --git a/acceleration/internal/store/connections.go b/acceleration/internal/store/connections.go index 5ad6418fb..79f522185 100644 --- a/acceleration/internal/store/connections.go +++ b/acceleration/internal/store/connections.go @@ -727,10 +727,10 @@ func (s *Store) ConnectorConnectionReferenced(ctx context.Context, customerID, i return false, stack.Wrap(errors.New("store: a customer and a connection id are required")) } var referenced bool - err := s.db.QueryRowContext(ctx, ` -SELECT `+boundByConfig+` -FROM connector_connections AS cc -WHERE cc.customer_id = ? AND cc.id = ? AND cc.deleted_at IS NULL`, customerID, id).Scan(&referenced) + err := s.db.NewSelect().Model((*ConnectorConnection)(nil)). + ColumnExpr(boundByConfig). + Where("cc.customer_id = ? AND cc.id = ? AND cc.deleted_at IS NULL", customerID, id). + Scan(ctx, &referenced) if errors.Is(err, sql.ErrNoRows) { return false, stack.Wrap(fmt.Errorf("%w: %s", ErrNoConnectorConnection, id)) } @@ -806,15 +806,12 @@ func (s *Store) ConsumeConnectorAuthorizationAttempt(ctx context.Context, state return ConnectorAuthorizationAttempt{}, errors.New("store: an oauth state is required") } var attempt ConnectorAuthorizationAttempt - err := s.db.NewRaw(` -UPDATE connector_authorization_attempts AS caa -SET consumed_at = now() -WHERE caa.state_hash = ? - AND caa.consumed_at IS NULL - AND caa.expires_at > now() - AND EXISTS (SELECT 1 FROM connector_connections AS cc - WHERE cc.id = caa.connection_id AND cc.customer_id = caa.customer_id AND cc.deleted_at IS NULL) -RETURNING caa.*`, AuthorizationStateHash(state)).Scan(ctx, &attempt) + err := s.db.NewUpdate().Model(&attempt). + Set("consumed_at = now()"). + Where("caa.state_hash = ?", AuthorizationStateHash(state)). + ApplyQueryBuilder(attemptOpen). + Returning("caa.*"). + Scan(ctx) if errors.Is(err, sql.ErrNoRows) { return ConnectorAuthorizationAttempt{}, ErrNoAuthorizationAttempt } @@ -833,16 +830,13 @@ func (s *Store) HandOffConnectorAuthorizationAttempt(ctx context.Context, id str if id == "" || len(previous) == 0 || len(sealed) == 0 || kekVersion < 1 { return errors.New("store: a handoff needs an attempt id, the blob it read and a new one under a key version of 1 or more") } - result, err := s.db.NewRaw(` -UPDATE connector_authorization_attempts AS caa -SET attempt_sealed = ?, kek_version = ? -WHERE caa.id = ? - AND caa.attempt_sealed = ? - AND caa.consumed_at IS NULL - AND caa.expires_at > now() - AND EXISTS (SELECT 1 FROM connector_connections AS cc - WHERE cc.id = caa.connection_id AND cc.customer_id = caa.customer_id AND cc.deleted_at IS NULL)`, - sealed, kekVersion, id, previous).Exec(ctx) + result, err := s.db.NewUpdate().Model((*ConnectorAuthorizationAttempt)(nil)). + Set("attempt_sealed = ?", sealed). + Set("kek_version = ?", kekVersion). + Where("caa.id = ?", id). + Where("caa.attempt_sealed = ?", previous). + ApplyQueryBuilder(attemptOpen). + Exec(ctx) if err != nil { return fmt.Errorf("store: hand off authorization attempt: %w", err) } @@ -863,15 +857,21 @@ func AuthorizationStateHash(state string) string { return hex.EncodeToString(hash[:]) } +// attemptOpen keeps the attempts that are unconsumed, unexpired and for a live connection. +func attemptOpen(q bun.QueryBuilder) bun.QueryBuilder { + return q. + Where("caa.consumed_at IS NULL"). + Where("caa.expires_at > now()"). + Where("EXISTS (SELECT 1 FROM connector_connections AS cc WHERE cc.id = caa.connection_id AND cc.customer_id = caa.customer_id AND cc.deleted_at IS NULL)") +} + // openAttempt is the one attempt matching where that is unconsumed, unexpired and for a // live connection. func (s *Store) openAttempt(ctx context.Context, where string, arg any) (ConnectorAuthorizationAttempt, error) { var attempt ConnectorAuthorizationAttempt err := s.db.NewSelect().Model(&attempt). Where(where, arg). - Where("caa.consumed_at IS NULL"). - Where("caa.expires_at > now()"). - Where("EXISTS (SELECT 1 FROM connector_connections AS cc WHERE cc.id = caa.connection_id AND cc.customer_id = caa.customer_id AND cc.deleted_at IS NULL)"). + ApplyQueryBuilder(attemptOpen). Scan(ctx) if errors.Is(err, sql.ErrNoRows) { return ConnectorAuthorizationAttempt{}, ErrNoAuthorizationAttempt diff --git a/acceleration/internal/store/connectors.go b/acceleration/internal/store/connectors.go index 91ad1d8bd..68371eca1 100644 --- a/acceleration/internal/store/connectors.go +++ b/acceleration/internal/store/connectors.go @@ -637,9 +637,10 @@ func (s *Store) DeleteConnectorDefinition(ctx context.Context, customerID, id st } // The deliveries go with their destination (ON DELETE CASCADE, // 20261007040000_connector_event_destinations.sql). - for _, table := range []string{"connector_oauth_clients", "connector_config_tokens", "connector_event_destinations"} { - if _, err := tx.ExecContext(ctx, "DELETE FROM "+table+" WHERE customer_id = ? AND connector_id = ?", customerID, id); err != nil { - return fmt.Errorf("store: delete the connector's %s: %w", table, err) + for _, model := range []any{(*ConnectorOAuthClient)(nil), (*ConnectorConfigToken)(nil), (*EventDestination)(nil)} { + q := tx.NewDelete().Model(model).Where("customer_id = ? AND connector_id = ?", customerID, id) + if _, err := q.Exec(ctx); err != nil { + return fmt.Errorf("store: delete the connector's %s: %w", q.GetTableName(), err) } } _, err = tx.NewDelete().Model((*ConnectorDefinition)(nil)). diff --git a/acceleration/internal/store/episodes.go b/acceleration/internal/store/episodes.go index ca37ef86b..159cd8931 100644 --- a/acceleration/internal/store/episodes.go +++ b/acceleration/internal/store/episodes.go @@ -309,24 +309,22 @@ func (s *Store) ClaimEpisodeSummaries(ctx context.Context, now time.Time, limit return claimed, nil } -// finishEpisodeSummaryQuery sets an ended episode's status and drops its lease, in one -// statement. -const finishEpisodeSummaryQuery = ` -WITH finished AS ( - UPDATE episodes SET status = ? - WHERE customer_id = ? AND id = ? AND status = 'ended' - RETURNING id -) -UPDATE episode_activity AS act SET summary_lease_until = NULL -FROM finished WHERE act.episode_id = finished.id` - // FinishEpisodeSummary sets an ended episode summarized or summary_failed and drops its // lease. An episode no longer ended, finished already, is left as it is. func (s *Store) FinishEpisodeSummary(ctx context.Context, customerID, id, status string) error { if status != EpisodeSummarized && status != EpisodeSummaryFailed { return stack.Wrap(fmt.Errorf("store: %q is not a status a summary finishes with", status)) } - _, err := s.db.NewRaw(finishEpisodeSummaryQuery, status, customerID, id).Exec(ctx) + finished := s.db.NewUpdate().Model((*Episode)(nil)). + Set("status = ?", status). + Where("ep.customer_id = ? AND ep.id = ? AND ep.status = ?", customerID, id, EpisodeEnded). + Returning("ep.id") + _, err := s.db.NewUpdate().With("finished", finished). + TableExpr("episode_activity AS act"). + TableExpr("finished"). + Set("summary_lease_until = NULL"). + Where("act.episode_id = finished.id"). + Exec(ctx) if err != nil { return stack.Wrap(fmt.Errorf("store: finish episode summary: %w", err)) } diff --git a/acceleration/internal/store/event_destinations.go b/acceleration/internal/store/event_destinations.go index af2c6c4e3..a32f5dea7 100644 --- a/acceleration/internal/store/event_destinations.go +++ b/acceleration/internal/store/event_destinations.go @@ -162,16 +162,16 @@ func (s *Store) RotateEventDestinationSecret(ctx context.Context, customerID, co } now := time.Now().UTC().Truncate(time.Microsecond) var destination EventDestination - err := s.db.NewRaw(` -UPDATE connector_event_destinations -SET previous_secret_sealed = secret_sealed, - previous_kek_version = kek_version, - previous_until = ?, - secret_sealed = ?, - kek_version = ?, - updated_at = ? -WHERE id = ? AND customer_id = ? AND connector_id = ? -RETURNING *`, previousUntil.UTC(), sealed, version, now, id, customerID, connectorID).Scan(ctx, &destination) + err := s.db.NewUpdate().Model(&destination). + Set("previous_secret_sealed = secret_sealed"). + Set("previous_kek_version = kek_version"). + Set("previous_until = ?", previousUntil.UTC()). + Set("secret_sealed = ?", sealed). + Set("kek_version = ?", version). + Set("updated_at = ?", now). + Where("id = ? AND customer_id = ? AND connector_id = ?", id, customerID, connectorID). + Returning("*"). + Scan(ctx) if errors.Is(err, sql.ErrNoRows) { return EventDestination{}, stack.Wrap(fmt.Errorf("%w: %s", ErrNoEventDestination, id)) } @@ -316,7 +316,7 @@ func (s *Store) claimEventDeliveries(ctx context.Context, query string, now time // end of their lease. found is false when no delivery is queued at all. func (s *Store) NextEventDeliveryAt(ctx context.Context) (next time.Time, found bool, err error) { var at sql.NullTime - if err := s.db.NewRaw("SELECT min(next_attempt_at) FROM connector_event_deliveries").Scan(ctx, &at); err != nil { + if err := s.db.NewSelect().Model((*EventDelivery)(nil)).ColumnExpr("min(next_attempt_at)").Scan(ctx, &at); err != nil { return time.Time{}, false, stack.Wrap(fmt.Errorf("store: next event delivery: %w", err)) } return at.Time, at.Valid, nil diff --git a/acceleration/internal/store/forget.go b/acceleration/internal/store/forget.go index dfd0ebe5a..d59832605 100644 --- a/acceleration/internal/store/forget.go +++ b/acceleration/internal/store/forget.go @@ -66,8 +66,11 @@ func (s *Store) RenameUserRequests(ctx context.Context, appID, userID, replaceme func (s *Store) UserOrganizations(ctx context.Context, appID string, userIDs ...string) ([]string, error) { shards, err := everyShard(ctx, s, func(ctx context.Context, db bun.IDB) ([]string, error) { var found []string - err := db.NewRaw(`SELECT DISTINCT organization_id FROM requests WHERE customer_id = ? AND user_id IN (?)`, - appID, bun.In(userIDs)).Scan(ctx, &found) + err := db.NewSelect().Model((*Request)(nil)). + Distinct(). + Column("r.organization_id"). + Where("r.customer_id = ? AND r.user_id IN (?)", appID, bun.In(userIDs)). + Scan(ctx, &found) return found, err }) if err != nil { @@ -75,12 +78,14 @@ func (s *Store) UserOrganizations(ctx context.Context, appID string, userIDs ... } var shared []string users := bun.In(userIDs) - if err := s.db.NewRaw(`SELECT organization_id FROM app_organizations WHERE app_id = ? - UNION SELECT organization_id FROM apps WHERE id = ? - UNION SELECT organization_id FROM limits WHERE level = ? AND app_id = ? AND user_id IN (?) - UNION SELECT organization_id FROM usage_resets WHERE app_id = ? AND user_id IN (?) - UNION SELECT organization_id FROM limit_grants WHERE app_id = ? AND user_id IN (?)`, - appID, appID, LevelUser, appID, users, appID, users, appID, users).Scan(ctx, &shared); err != nil { + err = s.db.NewSelect().Model((*AppOrganization)(nil)).Column("organization_id").Where("app_id = ?", appID). + Union(s.db.NewSelect().Model((*App)(nil)).Column("organization_id").Where("id = ?", appID)). + Union(s.db.NewSelect().Model((*Limit)(nil)).Column("organization_id"). + Where("level = ? AND app_id = ? AND user_id IN (?)", LevelUser, appID, users)). + Union(s.db.NewSelect().Model((*UsageReset)(nil)).Column("organization_id").Where("app_id = ? AND user_id IN (?)", appID, users)). + Union(s.db.NewSelect().Model((*LimitGrant)(nil)).Column("organization_id").Where("app_id = ? AND user_id IN (?)", appID, users)). + Scan(ctx, &shared) + if err != nil { return nil, stack.Wrap(fmt.Errorf("store: user organizations: %w", err)) } organizations := map[string]bool{} @@ -121,16 +126,15 @@ func (s *Store) ForgetUserSettings(ctx context.Context, appID, userID, replaceme err := s.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { for _, change := range []struct { count *int64 - query string - args []any + exec func(context.Context, ...any) (sql.Result, error) }{ - {&counts.Policies, "DELETE FROM policies WHERE scope = ? AND scope_id = ?", []any{ScopeUser, UserScopeID(appID, userID)}}, - {&counts.Limits, "DELETE FROM limits WHERE level = ? AND app_id = ? AND user_id = ?", []any{LevelUser, appID, userID}}, - {&counts.Resets, "UPDATE usage_resets SET user_id = ? WHERE app_id = ? AND user_id = ?", []any{replacement, appID, userID}}, - {&counts.Grants, "UPDATE limit_grants SET user_id = ? WHERE app_id = ? AND user_id = ?", []any{replacement, appID, userID}}, - {&counts.Campaigns, "UPDATE campaigns SET user_id = ? WHERE customer_id = ? AND user_id = ?", []any{replacement, appID, userID}}, + {&counts.Policies, tx.NewDelete().Model((*Policy)(nil)).Where("scope = ? AND scope_id = ?", ScopeUser, UserScopeID(appID, userID)).Exec}, + {&counts.Limits, tx.NewDelete().Model((*Limit)(nil)).Where("level = ? AND app_id = ? AND user_id = ?", LevelUser, appID, userID).Exec}, + {&counts.Resets, tx.NewUpdate().Model((*UsageReset)(nil)).Set("user_id = ?", replacement).Where("app_id = ? AND user_id = ?", appID, userID).Exec}, + {&counts.Grants, tx.NewUpdate().Model((*LimitGrant)(nil)).Set("user_id = ?", replacement).Where("app_id = ? AND user_id = ?", appID, userID).Exec}, + {&counts.Campaigns, tx.NewUpdate().Model((*Campaign)(nil)).Set("user_id = ?", replacement).Where("customer_id = ? AND user_id = ?", appID, userID).Exec}, } { - result, err := tx.ExecContext(ctx, change.query, change.args...) + result, err := change.exec(ctx) if err != nil { return err } diff --git a/acceleration/internal/store/grants.go b/acceleration/internal/store/grants.go index 572671c64..03e5ee4de 100644 --- a/acceleration/internal/store/grants.go +++ b/acceleration/internal/store/grants.go @@ -402,12 +402,17 @@ func addGrantUses(ctx context.Context, tx bun.Tx, uses []GrantUse) (int, error) for i, use := range uses { grants[i], usageIDs[i], micros[i] = use.GrantID, use.UsageID.String(), use.Micros } - if _, err := tx.ExecContext(ctx, "SELECT id FROM limit_grants WHERE id = ANY(?::bigint[]) ORDER BY id FOR UPDATE", - pgdialect.Array(slices.Compact(slices.Sorted(slices.Values(grants))))); err != nil { + _, err := tx.NewSelect().Model((*LimitGrant)(nil)). + Column("lg.id"). + Where("lg.id = ANY(?::bigint[])", pgdialect.Array(slices.Compact(slices.Sorted(slices.Values(grants))))). + Order("lg.id"). + For("UPDATE"). + Exec(ctx) + if err != nil { return 0, err } var added int - err := tx.NewRaw(` + err = tx.NewRaw(` WITH sent AS ( SELECT * FROM unnest(?::bigint[], ?::uuid[], ?::bigint[]) AS s (grant_id, usage_id, micros) ), counted AS ( diff --git a/acceleration/internal/store/image_jobs.go b/acceleration/internal/store/image_jobs.go index c2057453d..039e253f3 100644 --- a/acceleration/internal/store/image_jobs.go +++ b/acceleration/internal/store/image_jobs.go @@ -122,19 +122,20 @@ func (s *Store) ClaimImageJobs(ctx context.Context, now time.Time, limit int, le if limit < 1 { return claimed, nil } - err := s.db.NewRaw(` -WITH due AS ( - SELECT ij.id FROM image_jobs AS ij - WHERE ij.state = 'queued' AND ij.next_attempt_at <= ? AND (ij.message <> '') = ? - ORDER BY ij.next_attempt_at, ij.id - LIMIT ? - FOR UPDATE OF ij SKIP LOCKED -) -UPDATE image_jobs AS ij -SET next_attempt_at = ?, attempts = ij.attempts + 1 -FROM due -WHERE ij.id = due.id -RETURNING `+imageJobClaimed, now.UTC(), decided, limit, leaseUntil.UTC()).Scan(ctx, &claimed) + due := s.db.NewSelect().Model((*ImageJob)(nil)). + Column("ij.id"). + Where("ij.state = 'queued' AND ij.next_attempt_at <= ?", now.UTC()). + Where("(ij.message <> '') = ?", decided). + Order("ij.next_attempt_at", "ij.id"). + Limit(limit). + For("UPDATE OF ij SKIP LOCKED") + err := s.db.NewUpdate().With("due", due).Model((*ImageJob)(nil)). + TableExpr("due"). + Set("next_attempt_at = ?", leaseUntil.UTC()). + Set("attempts = ij.attempts + 1"). + Where("ij.id = due.id"). + Returning(imageJobClaimed). + Scan(ctx, &claimed) if err != nil { return nil, stack.Wrap(fmt.Errorf("store: claim image jobs: %w", err)) } diff --git a/acceleration/internal/store/oauth_clients.go b/acceleration/internal/store/oauth_clients.go index f9fa226f7..739bc82ca 100644 --- a/acceleration/internal/store/oauth_clients.go +++ b/acceleration/internal/store/oauth_clients.go @@ -114,25 +114,21 @@ func (s *Store) PutConnectorOAuthClient(ctx context.Context, client *ConnectorOA return fmt.Errorf("%w: %s lists %v, not %s", ErrOAuthClientRegistrationNotListed, client.ConnectorID, definition.Manifest.Client.Registration, client.Registration) } - err = tx.NewRaw(` -INSERT INTO connector_oauth_clients AS coc - (customer_id, connector_id, registration, client_id, auth_method, secret_sealed, kek_version, - provider_app_id, stream_app_pk, signing_secret_sealed, signing_kek_version, created_at, updated_at) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) -ON CONFLICT (customer_id, connector_id) DO UPDATE -SET client_id = EXCLUDED.client_id, - auth_method = EXCLUDED.auth_method, - secret_sealed = EXCLUDED.secret_sealed, - kek_version = EXCLUDED.kek_version, - provider_app_id = EXCLUDED.provider_app_id, - signing_secret_sealed = EXCLUDED.signing_secret_sealed, - signing_kek_version = EXCLUDED.signing_kek_version, - updated_at = EXCLUDED.updated_at -WHERE coc.registration = EXCLUDED.registration -RETURNING coc.created_at, coc.stream_app_pk`, - client.CustomerID, client.ConnectorID, client.Registration, client.ClientID, client.AuthMethod, - client.SecretSealed, client.KEKVersion, client.ProviderAppID, nullablePin(client.StreamAppPK), - client.SigningSecretSealed, client.SigningKEKVersion, now, now).Scan(ctx, &createdAt, &pin) + row := *client + row.CreatedAt, row.UpdatedAt = now, now + err = tx.NewInsert().Model(&row). + On("CONFLICT (customer_id, connector_id) DO UPDATE"). + Set("client_id = EXCLUDED.client_id"). + Set("auth_method = EXCLUDED.auth_method"). + Set("secret_sealed = EXCLUDED.secret_sealed"). + Set("kek_version = EXCLUDED.kek_version"). + Set("provider_app_id = EXCLUDED.provider_app_id"). + Set("signing_secret_sealed = EXCLUDED.signing_secret_sealed"). + Set("signing_kek_version = EXCLUDED.signing_kek_version"). + Set("updated_at = EXCLUDED.updated_at"). + Where("coc.registration = EXCLUDED.registration"). + Returning("coc.created_at, coc.stream_app_pk"). + Scan(ctx, &createdAt, &pin) // The conflict's WHERE kept the row, so nothing was written or returned. if errors.Is(err, sql.ErrNoRows) { return fmt.Errorf("%w: %s", ErrOAuthClientRegistration, client.ConnectorID) diff --git a/acceleration/internal/store/policies_test.go b/acceleration/internal/store/policies_test.go index 666be8954..9071a1180 100644 --- a/acceleration/internal/store/policies_test.go +++ b/acceleration/internal/store/policies_test.go @@ -3,6 +3,7 @@ package store import ( + "slices" "time" "github.com/google/uuid" @@ -155,6 +156,28 @@ func (s *StoreSuite) TestAnAppIsInTheOrganizationItIsRegisteredOrSeenUnder() { } } +func (s *StoreSuite) TestAnOrganizationsAppsAreTheOnesSeenOrRegisteredUnderItAndItsOwn() { + org := Organization{Name: "registering"} + s.Require().NoError(s.store.CreateOrganization(s.ctx, &org)) + registered := App{OrganizationID: org.ID, Name: "registered"} + s.Require().NoError(s.store.CreateApp(s.ctx, ®istered)) + moved := App{OrganizationID: org.ID, Name: "moved"} + s.Require().NoError(s.store.CreateApp(s.ctx, &moved)) + s.Require().NoError(s.store.JoinOrganization(s.ctx, moved.ID, "org-other")) + s.Require().NoError(s.store.JoinOrganization(s.ctx, "app-seen-b", org.ID)) + s.Require().NoError(s.store.JoinOrganization(s.ctx, "app-seen-a", org.ID)) + s.Require().NoError(s.store.JoinOrganization(s.ctx, "app-elsewhere", "org-other")) + + apps, err := s.store.OrganizationApps(s.ctx, org.ID) + + s.Require().NoError(err) + s.Require().NotEmpty(apps) + s.Equal(org.ID, apps[0], "its own id first") + s.ElementsMatch([]string{org.ID, registered.ID, "app-seen-a", "app-seen-b"}, apps, + "not the app a request named another organization for, nor another organization's") + s.Less(slices.Index(apps, "app-seen-a"), slices.Index(apps, "app-seen-b"), "in order") +} + func scopeIDs(policies []Policy) []string { ids := make([]string, len(policies)) for i, policy := range policies { diff --git a/acceleration/internal/store/price_changes.go b/acceleration/internal/store/price_changes.go index b8ef9d895..42e0bd717 100644 --- a/acceleration/internal/store/price_changes.go +++ b/acceleration/internal/store/price_changes.go @@ -258,8 +258,8 @@ func (s *Store) CreatePriceChange(ctx context.Context, write PriceChangeWrite, return err } } - var seeded bool - if err := tx.QueryRowContext(ctx, "SELECT EXISTS (SELECT 1 FROM price_changes)").Scan(&seeded); err != nil { + seeded, err := tx.NewSelect().Model((*PriceChange)(nil)).Exists(ctx) + if err != nil { return err } if !seeded { @@ -289,8 +289,8 @@ func (s *Store) CreatePriceChange(ctx context.Context, write PriceChangeWrite, CreatedBy: write.ActorID, IdempotencyKey: write.IdempotencyKey, } - var taken bool - if err := tx.QueryRowContext(ctx, "SELECT EXISTS (SELECT 1 FROM price_changes WHERE name = ?)", change.Name).Scan(&taken); err != nil { + taken, err := tx.NewSelect().Model((*PriceChange)(nil)).Where("name = ?", change.Name).Exists(ctx) + if err != nil { return err } if taken { diff --git a/acceleration/internal/store/prices.go b/acceleration/internal/store/prices.go index b2d5a1279..60939a043 100644 --- a/acceleration/internal/store/prices.go +++ b/acceleration/internal/store/prices.go @@ -417,13 +417,13 @@ func (s *Store) ActivePriceEntries(ctx context.Context, at time.Time) ([]PriceEn func activePriceEntries(ctx context.Context, db bun.IDB, at time.Time) ([]PriceEntry, error) { var newest []PriceEntry - err := db.NewRaw(` -SELECT DISTINCT ON (pe.modality, pe.provider, pe.model, pe.region, pe.byok, pe.attributes) `+qualified("pe", priceEntryColumns)+` -FROM price_list_entries AS pe -JOIN price_changes AS pc ON pc.name = pe.change AND pc.cancelled_at IS NULL -WHERE pe.effective_at <= ? -ORDER BY pe.modality, pe.provider, pe.model, pe.region, pe.byok, pe.attributes, pe.effective_at DESC, pe.change DESC, pe.id DESC`, at). - Scan(ctx, &newest) + err := db.NewSelect().Model(&newest). + DistinctOn("pe.modality, pe.provider, pe.model, pe.region, pe.byok, pe.attributes"). + Join("JOIN price_changes AS pc ON pc.name = pe.change AND pc.cancelled_at IS NULL"). + Where("pe.effective_at <= ?", at). + Order("pe.modality", "pe.provider", "pe.model", "pe.region", "pe.byok", "pe.attributes"). + OrderExpr("pe.effective_at DESC, pe.change DESC, pe.id DESC"). + Scan(ctx) if err != nil { return nil, stack.Wrap(fmt.Errorf("store: active price entries: %w", err)) } @@ -436,15 +436,6 @@ ORDER BY pe.modality, pe.provider, pe.model, pe.region, pe.byok, pe.attributes, return active, nil } -// qualified is a list of columns each prefixed with the table's alias. -func qualified(alias, columns string) string { - names := strings.Split(columns, ", ") - for i, name := range names { - names[i] = alias + "." + name - } - return strings.Join(names, ", ") -} - // NextPriceChangeAt is when the next change not cancelled takes effect after after, and // whether one is scheduled, so a list in memory can switch at that moment. func (s *Store) NextPriceChangeAt(ctx context.Context, after time.Time) (time.Time, bool, error) { diff --git a/acceleration/internal/store/shipping.go b/acceleration/internal/store/shipping.go index 7dfc110f0..1971d5eb2 100644 --- a/acceleration/internal/store/shipping.go +++ b/acceleration/internal/store/shipping.go @@ -142,9 +142,12 @@ func boundShipping(ctx context.Context, tx bun.Tx) error { func (s *Store) Unshipped(ctx context.Context) (int64, time.Duration, error) { var rows int64 var oldest float64 - err := s.shard.QueryRowContext(ctx, - "SELECT count(*), COALESCE(EXTRACT(EPOCH FROM now() - min(started_at)), 0)::float8 FROM requests WHERE to_ship AND organization_id <> ''", - ).Scan(&rows, &oldest) + err := s.shard.NewSelect().Model((*Request)(nil)). + ColumnExpr("count(*)"). + ColumnExpr("COALESCE(EXTRACT(EPOCH FROM now() - min(r.started_at)), 0)::float8"). + Where("r.to_ship"). + Where("r.organization_id <> ''"). + Scan(ctx, &rows, &oldest) if err != nil { return 0, 0, stack.Wrap(fmt.Errorf("store: unshipped: %w", err)) } @@ -168,21 +171,27 @@ func (s *Store) Reship(ctx context.Context, since time.Time, rejected bool) (int if !last.Valid { return 0, nil } - filter, args := "shipped_at IS NOT NULL AND shipped_at < ?", []any{started} - if rejected { - filter, args = "ship_error <> ''", nil - } var queued int64 for { + batch := s.shard.NewSelect().Model((*Request)(nil)). + Column("r.id"). + Where("r.id > ? AND r.started_at >= ?", last.Int64, since). + Where("r.usage_id IS NOT NULL AND NOT r.to_ship"). + Order("r.id"). + Limit(reshipBatch) + if rejected { + batch.Where("r.ship_error <> ''") + } else { + batch.Where("r.shipped_at IS NOT NULL AND r.shipped_at < ?", started) + } var ids []int64 - err := s.shard.NewRaw(` -WITH batch AS ( - SELECT id FROM requests - WHERE id > ? AND started_at >= ? AND usage_id IS NOT NULL AND NOT to_ship AND `+filter+` - ORDER BY id LIMIT ? -) -UPDATE requests AS r SET to_ship = true, ship_error = '' FROM batch WHERE r.id = batch.id -RETURNING r.id`, append(append([]any{last.Int64, since}, args...), reshipBatch)...).Scan(ctx, &ids) + err := s.shard.NewUpdate().With("batch", batch).Model((*Request)(nil)). + TableExpr("batch"). + Set("to_ship = true"). + Set("ship_error = ''"). + Where("r.id = batch.id"). + Returning("r.id"). + Scan(ctx, &ids) if err != nil { return queued, stack.Wrap(fmt.Errorf("store: reship: %w", err)) } diff --git a/acceleration/internal/store/shipping_test.go b/acceleration/internal/store/shipping_test.go index 123f0a705..036e89f63 100644 --- a/acceleration/internal/store/shipping_test.go +++ b/acceleration/internal/store/shipping_test.go @@ -29,6 +29,76 @@ func (s *StoreSuite) TestMarkingARowAnotherSessionHoldsFailsWithinTheLockWait() s.ErrorContains(err, "lock timeout", "the sweeper's other write waits no longer") } +// shippable records a row of organization's started at at, still to be shipped. +func (s *StoreSuite) shippable(organization string, at time.Time) uuid.UUID { + request := &Request{UsageID: uuid.Must(uuid.NewV7()), Modality: "llm", CustomerID: "acme", OrganizationID: organization, + Provider: "openai", Model: "gpt", StartedAt: at, Success: true, ToShip: true} + s.Require().NoError(s.store.RecordRequest(s.ctx, request)) + return request.UsageID +} + +// shipping is the row's to_ship and ship_error. +func (s *StoreSuite) shipping(usageID uuid.UUID) (bool, string) { + var request Request + s.Require().NoError(s.store.ShardDB().NewSelect().Model(&request).Where("usage_id = ?", usageID).Scan(s.ctx)) + return request.ToShip, request.ShipError +} + +func (s *StoreSuite) TestUnshippedCountsTheRowsWithAnOrganizationStillToShip() { + now := time.Now().UTC() + s.shippable("org", now.Add(-2*time.Hour)) + s.shippable("org", now.Add(-time.Hour)) + s.shippable("", now.Add(-3*time.Hour)) + shipped := s.shippable("org", now.Add(-4*time.Hour)) + s.Require().NoError(s.store.RecordShipment(s.ctx, []uuid.UUID{shipped}, nil)) + + rows, oldest, err := s.store.Unshipped(s.ctx) + + s.Require().NoError(err) + s.EqualValues(2, rows, "neither the shipped row nor the one with no organization") + s.InDelta((2 * time.Hour).Seconds(), oldest.Seconds(), 60) +} + +func (s *StoreSuite) TestReshipQueuesTheShippedRowsStartedSinceAgain() { + since := time.Now().UTC().Add(-time.Hour) + recent := s.shippable("org", since.Add(time.Minute)) + older := s.shippable("org", since.Add(-time.Minute)) + refused := s.shippable("org", since.Add(2*time.Minute)) + waiting := s.shippable("org", since.Add(3*time.Minute)) + s.Require().NoError(s.store.RecordShipment(s.ctx, []uuid.UUID{recent, older}, map[uuid.UUID]string{refused: "unknown price"})) + + queued, err := s.store.Reship(s.ctx, since, false) + + s.Require().NoError(err) + s.EqualValues(1, queued) + toShip, _ := s.shipping(recent) + s.True(toShip) + toShip, _ = s.shipping(older) + s.False(toShip, "started before since") + toShip, reason := s.shipping(refused) + s.False(toShip, "the service refused it, it never had it") + s.Equal("unknown price", reason) + toShip, _ = s.shipping(waiting) + s.True(toShip, "still queued, as it was") +} + +func (s *StoreSuite) TestReshipOfRejectedRowsQueuesTheRefusedOnesAndClearsWhy() { + since := time.Now().UTC().Add(-time.Hour) + shipped := s.shippable("org", since.Add(time.Minute)) + refused := s.shippable("org", since.Add(2*time.Minute)) + s.Require().NoError(s.store.RecordShipment(s.ctx, []uuid.UUID{shipped}, map[uuid.UUID]string{refused: "unknown price"})) + + queued, err := s.store.Reship(s.ctx, since, true) + + s.Require().NoError(err) + s.EqualValues(1, queued) + toShip, reason := s.shipping(refused) + s.True(toShip) + s.Empty(reason) + toShip, _ = s.shipping(shipped) + s.False(toShip, "the service has it") +} + func (s *StoreSuite) TestPostgresProbesTheConnectionHoldingTheShipLock() { lock := s.store.ShipLock() defer lock.Release() diff --git a/acceleration/internal/store/store.go b/acceleration/internal/store/store.go index 4a48acbfc..9870fe234 100644 --- a/acceleration/internal/store/store.go +++ b/acceleration/internal/store/store.go @@ -876,12 +876,16 @@ func spendGroups(groupBy string, acrossApps bool) (value, join string, ok bool) func (s *Store) OrganizationApps(ctx context.Context, organizationID string) ([]string, error) { // One statement, so an app a request records under the organization meanwhile is in one // half or the other. + namedOther := s.db.NewSelect().Model((*AppOrganization)(nil)). + ColumnExpr("1"). + Where("ao.app_id = ap.id AND ao.organization_id <> ?", organizationID) + named := s.db.NewSelect().Model((*AppOrganization)(nil)).Column("app_id").Where("organization_id = ?", organizationID). + Union(s.db.NewSelect().Model((*App)(nil)). + Column("id"). + Where("ap.organization_id = ?", organizationID). + Where("NOT EXISTS (?)", namedOther)) var members []string - err := s.db.NewRaw(`SELECT app_id FROM app_organizations WHERE organization_id = ? -UNION -SELECT id FROM apps AS ap WHERE organization_id = ? - AND NOT EXISTS (SELECT 1 FROM app_organizations AS ao WHERE ao.app_id = ap.id AND ao.organization_id <> ?) -ORDER BY 1`, organizationID, organizationID, organizationID).Scan(ctx, &members) + err := s.db.NewSelect().TableExpr("(?) AS member", named).Column("member.app_id").OrderExpr("1").Scan(ctx, &members) if err != nil { return nil, stack.Wrap(fmt.Errorf("store: organization apps: %w", err)) } @@ -1092,15 +1096,16 @@ func (s *Store) CustomerTagKeys( if err != nil || read.rows == 0 { return read, err } - err = db.NewRaw(` -SELECT tag.key AS key, tag.value AS value, - SUM(r.cost_micros) AS cost_micros_total, `+requestCount+` AS request_count -FROM requests AS r -CROSS JOIN LATERAL jsonb_each_text(r.tags) AS tag(key, value) -WHERE r.customer_id = ? - AND r.started_at >= ? AND r.started_at < ? - AND r.tags @> ?::jsonb -GROUP BY tag.key, tag.value`, customerID, from, to, string(filter)).Scan(ctx, &read.values) + err = db.NewSelect().Model((*Request)(nil)). + ColumnExpr("tag.key AS key, tag.value AS value"). + ColumnExpr("SUM(r.cost_micros) AS cost_micros_total"). + ColumnExpr(requestCount+" AS request_count"). + Join("CROSS JOIN LATERAL jsonb_each_text(r.tags) AS tag(key, value)"). + Where("r.customer_id = ?", customerID). + Where("r.started_at >= ? AND r.started_at < ?", from, to). + Where("r.tags @> ?::jsonb", string(filter)). + Group("tag.key", "tag.value"). + Scan(ctx, &read.values) return read, err }) if err != nil { @@ -1606,15 +1611,16 @@ func (s *Store) CustomerTurns(ctx context.Context, customerID, configID string, } counts, err := everyShard(ctx, s, func(ctx context.Context, db bun.IDB) (SessionTurns, error) { var counts SessionTurns - err := db.NewRaw(` -SELECT COUNT(DISTINCT a.session_id) AS sessions, COUNT(*) AS turns -FROM agent_responses AS a -JOIN agent_sessions AS s ON s.id = a.session_id -WHERE a.customer_id = ?0 AND (?1 = '' OR s.config_id = ?1) - AND a.created_at >= ?2 AND a.created_at < ?3 - AND a.parent_response_id IS NULL AND a.rewound_at IS NULL`, - customerID, configID, from, to, - ).Scan(ctx, &counts) + q := db.NewSelect().Model((*AgentResponse)(nil)). + ColumnExpr("COUNT(DISTINCT ars.session_id) AS sessions, COUNT(*) AS turns"). + Join("JOIN agent_sessions AS asn ON asn.id = ars.session_id"). + Where("ars.customer_id = ?", customerID). + Where("ars.created_at >= ? AND ars.created_at < ?", from, to). + Where("ars.parent_response_id IS NULL AND ars.rewound_at IS NULL") + if configID != "" { + q.Where("asn.config_id = ?", configID) + } + err := q.Scan(ctx, &counts) return counts, err }) if err != nil { diff --git a/acceleration/internal/store/tool_pins.go b/acceleration/internal/store/tool_pins.go index 02cced618..47ce48161 100644 --- a/acceleration/internal/store/tool_pins.go +++ b/acceleration/internal/store/tool_pins.go @@ -124,9 +124,11 @@ func (s *Store) ReapproveConnectorTools(ctx context.Context, connection Connecto // FOR SHARE: a consent or a credentials write that begins a new grant updates this row, // so one that comes later waits for this transaction, and one that came first is seen. var sameGrant bool - err := tx.QueryRowContext(ctx, "SELECT connected_at IS NOT DISTINCT FROM ? FROM connector_connections "+ - "WHERE customer_id = ? AND id = ? AND deleted_at IS NULL FOR SHARE", - connection.ConnectedAt, connection.CustomerID, connection.ID).Scan(&sameGrant) + err := tx.NewSelect().Model((*ConnectorConnection)(nil)). + ColumnExpr("cc.connected_at IS NOT DISTINCT FROM ?", connection.ConnectedAt). + Where("cc.customer_id = ? AND cc.id = ? AND cc.deleted_at IS NULL", connection.CustomerID, connection.ID). + For("SHARE"). + Scan(ctx, &sameGrant) if errors.Is(err, sql.ErrNoRows) { return fmt.Errorf("%w: %s", ErrNoConnectorConnection, connection.ID) } diff --git a/acceleration/internal/store/user_phones.go b/acceleration/internal/store/user_phones.go index 6caf62340..e105d8f77 100644 --- a/acceleration/internal/store/user_phones.go +++ b/acceleration/internal/store/user_phones.go @@ -178,8 +178,12 @@ func (s *Store) LinkedPhone(ctx context.Context, customerID string, app int64, u Phone string `bun:"phone"` Since *time.Time `bun:"since"` } - err := s.db.NewRaw(`SELECT up.phone, `+historyBoundary+` AS since FROM user_phones AS up `+historyJoin+` -WHERE up.customer_id = ? AND up.stream_app_pk = ? AND up.user_id = ?`, customerID, s.linkApp(app), userID).Scan(ctx, &rows) + err := s.db.NewSelect().Model((*UserPhone)(nil)). + Column("up.phone"). + ColumnExpr(historyBoundary+" AS since"). + Join(historyJoin). + Where("up.customer_id = ? AND up.stream_app_pk = ? AND up.user_id = ?", customerID, s.linkApp(app), userID). + Scan(ctx, &rows) if err != nil { return "", nil, false, stack.Wrap(fmt.Errorf("store: linked phone: %w", err)) } @@ -411,15 +415,19 @@ const reachedAt = `regexp_replace(cm.address, '^` + retiredPrefix + `[^:]*:', '' // counts as on: it writes the same episodes, which a session reads once the config turns on. // In tx, under the link's lock. func (s *Store) retireHistory(ctx context.Context, tx bun.Tx, link UserPhone) error { - _, err := tx.NewRaw(`UPDATE contact_map AS cm -SET address = ?4 || cm.id || ':' || cm.address, updated_at = ?5 -WHERE cm.customer_id = ?0 - AND cm.agent_config_id IN (SELECT ac.id FROM agent_configs AS ac WHERE ac.customer_id = ?0 AND (ac.episode_cards OR ac.episode_cards_record)) - AND cm.kind = ?1 AND cm.address = ?2 - AND COALESCE(NULLIF(cm.stream_app_pk, ?3), 0) = ?6 - AND EXISTS (SELECT 1 FROM episodes AS ep WHERE ep.contact_id = cm.id)`, - link.CustomerID, ContactPhone, link.Phone, s.streamPins().deployment(), retiredPrefix, time.Now().UTC(), - link.StreamAppPK). + writesCards := tx.NewSelect().Model((*AgentConfig)(nil)). + Column("ac.id"). + Where("ac.customer_id = ?", link.CustomerID). + Where("ac.episode_cards OR ac.episode_cards_record") + withEpisode := tx.NewSelect().Model((*Episode)(nil)).ColumnExpr("1").Where("ep.contact_id = cm.id") + _, err := tx.NewUpdate().Model((*ContactMapEntry)(nil)). + Set("address = ? || cm.id || ':' || cm.address", retiredPrefix). + Set("updated_at = ?", time.Now().UTC()). + Where("cm.customer_id = ?", link.CustomerID). + Where("cm.agent_config_id IN (?)", writesCards). + Where("cm.kind = ? AND cm.address = ?", ContactPhone, link.Phone). + Where("COALESCE(NULLIF(cm.stream_app_pk, ?), 0) = ?", s.streamPins().deployment(), link.StreamAppPK). + Where("EXISTS (?)", withEpisode). Exec(ctx) if err != nil { return stack.Wrap(fmt.Errorf("store: retire a linked number's history: %w", err)) diff --git a/acceleration/internal/usage/ledger/forget.go b/acceleration/internal/usage/ledger/forget.go index 4ec298f6c..eb5161f80 100644 --- a/acceleration/internal/usage/ledger/forget.go +++ b/acceleration/internal/usage/ledger/forget.go @@ -25,17 +25,33 @@ func (l *Ledger) ForgetUser(ctx context.Context, forget usage.ForgetUser, actor var events, windows []counted err := l.db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { if forget.DryRun { - if err := tx.NewRaw(`SELECT organization_id, count(*) FROM usage_events WHERE app_id = ? AND user_id = ? - GROUP BY organization_id`, forget.AppID, forget.UserID).Scan(ctx, &events); err != nil { + err := tx.NewSelect().Model((*UsageEvent)(nil)). + Column("organization_id"). + ColumnExpr("count(*) AS count"). + Where("app_id = ? AND user_id = ?", forget.AppID, forget.UserID). + Group("organization_id"). + Scan(ctx, &events) + if err != nil { return err } - return tx.NewRaw(`SELECT organization_id, count(*) FROM usage_windows WHERE subject = ? AND app_id = ? AND user_id = ? - GROUP BY organization_id`, usage.SubjectUser, forget.AppID, forget.UserID).Scan(ctx, &windows) + return tx.NewSelect().Model((*UsageWindow)(nil)). + Column("organization_id"). + ColumnExpr("count(*) AS count"). + Where("subject = ? AND app_id = ? AND user_id = ?", usage.SubjectUser, forget.AppID, forget.UserID). + Group("organization_id"). + Scan(ctx, &windows) } - if err := tx.NewRaw(`WITH moved AS ( - UPDATE usage_events SET user_id = ? WHERE app_id = ? AND user_id = ? RETURNING organization_id) - SELECT organization_id, count(*) FROM moved GROUP BY organization_id`, - forget.Replacement, forget.AppID, forget.UserID).Scan(ctx, &events); err != nil { + moved := tx.NewUpdate().Model((*UsageEvent)(nil)). + Set("user_id = ?", forget.Replacement). + Where("app_id = ? AND user_id = ?", forget.AppID, forget.UserID). + Returning("organization_id") + err := tx.NewSelect().With("moved", moved). + Table("moved"). + Column("organization_id"). + ColumnExpr("count(*) AS count"). + Group("organization_id"). + Scan(ctx, &events) + if err != nil { return err } if err := tx.NewRaw(`WITH moved AS ( diff --git a/acceleration/internal/usage/ledger/ledger.go b/acceleration/internal/usage/ledger/ledger.go index 55005fd11..e7aad4a8e 100644 --- a/acceleration/internal/usage/ledger/ledger.go +++ b/acceleration/internal/usage/ledger/ledger.go @@ -383,8 +383,8 @@ func storedWindows(ctx context.Context, db bun.IDB, wanted []UsageWindow) ([]Usa func (l *Ledger) PruneWindows(ctx context.Context, cutoff time.Time) (int64, error) { var deleted int64 for { - result, err := l.db.NewRaw(`DELETE FROM usage_windows WHERE ctid IN ( - SELECT ctid FROM usage_windows WHERE window_end < ? LIMIT ?)`, cutoff, pruneBatch).Exec(ctx) + ended := l.db.NewSelect().Model((*UsageWindow)(nil)).Column("ctid").Where("window_end < ?", cutoff).Limit(pruneBatch) + result, err := l.db.NewDelete().Model((*UsageWindow)(nil)).Where("ctid IN (?)", ended).Exec(ctx) if err != nil { return deleted, fmt.Errorf("ledger: prune windows: %w", err) }