From b332d453cb6b2075c33b9cf900ce7a904431864d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 17:51:26 +0000 Subject: [PATCH 1/3] refactor(meter): usage reporting as a package every server can use What sends usage records to QUMO_USAGE_URL moves out of the relay's verifier into internal/meter: sessions (Open, Report, Close), running totals per key (Total), coalescing, batches of 500, retries through an outage, signing and the last send at shutdown. The relay keeps only its own decisions (a publisher by itself, viewers in one total per key) in internal/auth. The records it sends are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KPWMJL7PH3rPhK5UTGavaB --- CHANGELOG.md | 4 + internal/auth/usage.go | 320 +++------------------- internal/auth/usage_viewers_test.go | 7 +- internal/auth/verifier_test.go | 61 +---- internal/meter/meter.go | 399 ++++++++++++++++++++++++++++ internal/meter/meter_test.go | 305 +++++++++++++++++++++ 6 files changed, 757 insertions(+), 339 deletions(-) create mode 100644 internal/meter/meter.go create mode 100644 internal/meter/meter_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index b9e2dcde..3e145c78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **A relay signs its key-set and usage requests, and `qumo dns` its feed reads, with its peer identity (`internal/reqsign`, #601).** With `PEER_CERT_FILE` and `PEER_KEY_FILE` (an ECDSA key; `qumo dns` takes the same two settings), each request carries `X-Relay-Certificate`, `X-Relay-Timestamp` and `X-Relay-Signature`: the key's ECDSA signature (ASN.1, base64) over SHA-256 of `qumo-relay-request`, the method, the request URI (path and query), the timestamp and the hex SHA-256 of the body, one per line, signed again on every retry. The receiver then knows which relay is calling from a key only that relay holds, and a captured request can't be changed or used for long, where a bearer token copied from a node could be reused until it is replaced. The bearer token (`QUMO_RELAY_TOKEN`, `QUMO_DNS_FEED_TOKEN`) is still sent beside it. A 401 is logged with the server's `Date` beside the local time. Funnel, HLS and ingest, which have no peer identity, are unchanged. +### Changed + +- **Usage reporting is a package of its own, for every server to use (`internal/meter`).** What sends usage records (`usage.Record`) to `QUMO_USAGE_URL` is no longer the relay's: `meter.Reporter` takes a server's sessions (`Open`, `Report` with cumulative metrics, `Close`) and running totals per key (`Total`), and keeps them in memory, coalesces each session's usage, batches by 500, retries through an outage, signs the requests when given a signer and sends what is left at shutdown, as the relay did. The relay reports through it, with what it decides itself (a publisher by itself, viewers in one total per key) kept in `internal/auth`. The records the relay sends are unchanged. + ## [v0.20.261114] - 2026-11-14 ### Added diff --git a/internal/auth/usage.go b/internal/auth/usage.go index 5261d466..0d9d461a 100644 --- a/internal/auth/usage.go +++ b/internal/auth/usage.go @@ -1,38 +1,23 @@ package auth import ( - "bytes" "context" - "crypto/rand" - "encoding/json" - "errors" - "fmt" - "io" - "log/slog" "math" - "net/http" "sync" "time" + "github.com/qumo-dev/qumo/internal/meter" "github.com/qumo-dev/qumo/internal/reqsign" "github.com/qumo-dev/qumo/usage" ) const ( - // usageFlushInterval is how often pending usage records are sent. - usageFlushInterval = 10 * time.Second - // usageSendTimeout bounds one send, and the last send at shutdown. - usageSendTimeout = 10 * time.Second - // maxBatchRecords bounds one POST, so a backlog after an outage goes out - // in pieces a receiver's body limit accepts. - maxBatchRecords = 500 - // maxPendingEvents caps the opens and closes waiting while the usage - // receiver can't be reached; the oldest are dropped past it. Usage needs - // no cap: it is one cumulative record per live session. - maxPendingEvents = 50_000 // sessionGrace is how long a session is remembered after it was last // heard of, at connect or a revalidate, for an end that arrives late. sessionGrace = 5 * time.Minute + // forgetInterval is how often sessions whose end never came are looked + // for. + forgetInterval = 10 * time.Second ) // usageSession is what the reporter remembers of a verified session. @@ -51,44 +36,21 @@ type usageSession struct { counted Bytes } -// viewerTotal is the bytes of every viewer session of one key since the relay -// started, and how much of it the receiver has acknowledged. -type viewerTotal struct { - total Bytes - reported Bytes - sent bool // reported at least once -} - -// usageReporter sends usage records (package usage) to a usage URL -// (QUMO_USAGE_URL). +// usageReporter reports the relay's sessions to a usage URL +// (QUMO_USAGE_URL), through a meter.Reporter. // // A session that may publish is reported by itself: an open when it is // admitted, its cumulative bytes at each revalidate, and a close with the // final totals. Every other session is a viewer, added up per key into one // running total for this run of the relay, sent as a viewers record. No open // or close is sent for viewers. -// -// Every record is cumulative, so a receiver can take a resent one without -// counting it twice. Records wait in memory between sends and through an -// outage; per-session usage is coalesced to its latest report. type usageReporter struct { - url string - token string - signer *reqsign.Signer // nil: requests are not signed - client *http.Client - // refusals logs the control plane's 401s, the first of a run only. - refusals reqsign.Refusals - now func() time.Time - // run names this run of the relay in its viewers records: a restarted - // relay starts new totals rather than lowering old ones. - runID string + meter *meter.Reporter + now func() time.Time mu sync.Mutex sessions map[string]usageSession - events []usage.Record // opens and closes, in order - latest map[string]usage.Record // latest cumulative report per session - viewers map[string]*viewerTotal // by kid - failing bool // the last send failed + viewers map[string]Bytes // every viewer session's bytes this run, by kid // closed is when each session that ended was closed, until it is // forgotten as a stale one would be: an open that comes after its // close, from a ticket granted as the session ended, is ignored. @@ -96,19 +58,15 @@ type usageReporter struct { } func newUsageReporter(rawURL, bearer string, signer *reqsign.Signer) (*usageReporter, error) { - if err := checkURL(rawURL); err != nil { + m, err := meter.New(meter.Config{Name: "relay", URL: rawURL, Token: bearer, Signer: signer}) + if err != nil { return nil, err } return &usageReporter{ - url: rawURL, - token: bearer, - signer: signer, - client: reqsign.NoRedirectClient(usageSendTimeout), + meter: m, now: time.Now, - runID: rand.Text(), sessions: map[string]usageSession{}, - latest: map[string]usage.Record{}, - viewers: map[string]*viewerTotal{}, + viewers: map[string]Bytes{}, closed: map[string]time.Time{}, }, nil } @@ -124,15 +82,15 @@ func (r *usageReporter) open(id string, s usageSession) { s.seen = r.now() r.sessions[id] = s if s.publishes { - r.appendEvent(r.record(usage.SessionOpen, id, s)) + r.meter.Open(id, s.kid) } } // publishing records that a viewer session may now publish, such as by a // ticket it presented: it is reported by itself from now on, opening with a // session_open. What it moved as a viewer up to its last revalidate stays in -// its key's viewer total; what it moved since is reported as its usage. A session it doesn't know, or that publishes already, is left as it -// is. +// its key's viewer total; what it moved since is reported as its usage. A +// session it doesn't know, or that publishes already, is left as it is. func (r *usageReporter) publishing(id string) { r.mu.Lock() defer r.mu.Unlock() @@ -142,7 +100,7 @@ func (r *usageReporter) publishing(id string) { } s.publishes = true r.sessions[id] = s - r.appendEvent(r.record(usage.SessionOpen, id, s)) + r.meter.Open(id, s.kid) } // revalidated records a session's cumulative bytes at a revalidate and, when @@ -162,9 +120,7 @@ func (r *usageReporter) revalidated(id string, b Bytes, live bool) { case !s.publishes: s = r.countViewer(s, b) case b.Sent > s.counted.Sent || b.Received > s.counted.Received: - rec := r.record(usage.Usage, id, s) - rec.Metrics = usageMetrics(b.since(s.counted)) - r.latest[id] = rec + r.meter.Report(id, s.kid, usageMetrics(b.since(s.counted))) } r.sessions[id] = s } @@ -184,54 +140,28 @@ func (r *usageReporter) close(id string, b Bytes, reason string) bool { r.countViewer(s, b) return true } - delete(r.latest, id) // the close carries totals at least as large - rec := r.record(usage.SessionClose, id, s) - rec.Metrics = usageMetrics(b.since(s.counted)) - rec.Reason = reason - r.appendEvent(rec) + r.meter.Close(id, s.kid, usageMetrics(b.since(s.counted)), reason) return true } -// countViewer adds what a viewer session moved since it was last -// counted to its key's viewer total, and returns the session with that -// recorded. The caller holds mu. +// countViewer adds what a viewer session moved since it was last counted to +// its key's viewer total, and returns the session with that recorded. The +// caller holds mu. func (r *usageReporter) countViewer(s usageSession, b Bytes) usageSession { t := r.viewers[s.kid] - if t == nil { - t = &viewerTotal{} - r.viewers[s.kid] = t - } if b.Sent > s.counted.Sent { - t.total.Sent += b.Sent - s.counted.Sent + t.Sent += b.Sent - s.counted.Sent s.counted.Sent = b.Sent } if b.Received > s.counted.Received { - t.total.Received += b.Received - s.counted.Received + t.Received += b.Received - s.counted.Received s.counted.Received = b.Received } + r.viewers[s.kid] = t + r.meter.Total(s.kid, usageMetrics(t)) return s } -func (r *usageReporter) record(typ usage.Type, id string, s usageSession) usage.Record { - return usage.Record{Type: typ, SessionID: id, KID: s.kid, TS: r.timestamp()} -} - -// timestamp is a record's ts: now, to the second, as the reports have always -// carried it. -func (r *usageReporter) timestamp() time.Time { - return r.now().UTC().Truncate(time.Second) -} - -// appendEvent queues an open or close, dropping the oldest past the cap. -// The caller holds mu. -func (r *usageReporter) appendEvent(rec usage.Record) { - r.events = append(r.events, rec) - if over := len(r.events) - maxPendingEvents; over > 0 { - slog.Warn("relay: usage backlog full; dropping the oldest session events", "dropped", over) - r.events = r.events[over:] - } -} - func usageMetrics(b Bytes) map[string]int64 { return map[string]int64{usage.IngressBytes: clampInt64(b.Received), usage.EgressBytes: clampInt64(b.Sent)} } @@ -243,45 +173,28 @@ func clampInt64(v uint64) int64 { return int64(v) } -// run sends pending records every usageFlushInterval until ctx ends, then -// once more. +// run sends usage until ctx ends, then once more, forgetting sessions whose +// end never came as it goes. func (r *usageReporter) run(ctx context.Context) { - ticker := time.NewTicker(usageFlushInterval) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - last, cancel := context.WithTimeout(context.WithoutCancel(ctx), usageSendTimeout) - defer cancel() - if err := r.flush(last); err != nil { - slog.Error("relay: last usage send failed; records lost", "error", err) + var wg sync.WaitGroup + wg.Go(func() { r.meter.Run(ctx) }) + wg.Go(func() { + ticker := time.NewTicker(forgetInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + r.forgetStale() } - return - case <-ticker.C: - r.forgetStale() - r.logOutcome(ctx, r.flush(ctx)) } - } + }) + wg.Wait() } -// logOutcome logs a send's first failure as a warning, repeats quietly, and -// the recovery as info: during an outage the relay retries every few -// seconds, and one line says so. -func (r *usageReporter) logOutcome(ctx context.Context, err error) { - r.mu.Lock() - wasFailing := r.failing - r.failing = err != nil - r.mu.Unlock() - switch { - case err != nil && ctx.Err() != nil: - case err != nil && !wasFailing: - slog.Warn("relay: usage send failed; keeping the records and retrying", "error", err) - case err != nil: - slog.Debug("relay: usage send still failing", "error", err) - case wasFailing: - slog.Info("relay: usage send recovered") - } -} +// flush sends every pending record now. +func (r *usageReporter) flush(ctx context.Context) error { return r.meter.Flush(ctx) } // forgetStale drops sessions whose end never came: those not heard of for // sessionGrace. @@ -292,6 +205,7 @@ func (r *usageReporter) forgetStale() { for id, s := range r.sessions { if s.seen.Before(cutoff) { delete(r.sessions, id) + r.meter.Forget(id) } } for id, at := range r.closed { @@ -301,152 +215,6 @@ func (r *usageReporter) forgetStale() { } } -// errMalformed is a 400 or 422 answer to a usage batch: the receiver can't -// read it, so resending it can't succeed. -var errMalformed = errors.New("usage batch malformed") - -// flush sends every pending record: opens, then per-session usage, then the -// viewer totals that changed, then closes, in batches of at most -// maxBatchRecords. A batch the receiver can't read (400, 422) is dropped, so -// it can't block later records. Any other failure, a 401 or 413 included, -// stops the flush and puts the unsent records back, unless newer ones -// replaced them, to try again next time. -func (r *usageReporter) flush(ctx context.Context) error { - r.mu.Lock() - events, latest := r.events, r.latest - r.events, r.latest = nil, map[string]usage.Record{} - // A viewer total is sent when it has changed since the receiver last - // acknowledged it. It stays pending until then, so it needs no requeue. - totals := map[string]Bytes{} // by kid - var viewers []usage.Record - for kid, t := range r.viewers { - if t.sent && t.total == t.reported { - continue - } - viewers = append(viewers, usage.Record{ - Type: usage.Viewers, Run: r.runID, KID: kid, Metrics: usageMetrics(t.total), TS: r.timestamp(), - }) - totals[kid] = t.total - } - r.mu.Unlock() - if len(events) == 0 && len(latest) == 0 && len(viewers) == 0 { - return nil - } - - all := make([]usage.Record, 0, len(events)+len(latest)+len(viewers)) - for _, e := range events { - if e.Type == usage.SessionOpen { - all = append(all, e) - } - } - for _, u := range latest { - all = append(all, u) - } - all = append(all, viewers...) - for _, e := range events { - if e.Type == usage.SessionClose { - all = append(all, e) - } - } - - for start := 0; start < len(all); start += maxBatchRecords { - batch := all[start:min(start+maxBatchRecords, len(all))] - err := r.send(ctx, batch) - switch { - case err == nil: - r.acknowledge(batch, totals) - case errors.Is(err, errMalformed): - slog.Error("relay: usage batch refused as malformed; dropped", "records", len(batch), "error", err) - // Not resent as it is; a viewer total goes again when it grows. - r.acknowledge(batch, totals) - default: - r.requeue(all[start:]) - return err - } - } - return nil -} - -// acknowledge records the viewer totals in a sent batch, by kid in totals, as -// reported. -func (r *usageReporter) acknowledge(batch []usage.Record, totals map[string]Bytes) { - r.mu.Lock() - defer r.mu.Unlock() - for _, rec := range batch { - if rec.Type != usage.Viewers { - continue - } - if t := r.viewers[rec.KID]; t != nil { - t.reported, t.sent = totals[rec.KID], true - } - } -} - -// requeue puts unsent records back before anything recorded since: opens -// and closes in order, and a session's usage record only if the session is -// still live and has no newer one. Viewer totals aren't put back: they stay -// pending until acknowledged. -func (r *usageReporter) requeue(unsent []usage.Record) { - r.mu.Lock() - defer r.mu.Unlock() - var events []usage.Record - for _, rec := range unsent { - switch rec.Type { - case usage.Viewers: - continue - case usage.SessionOpen, usage.SessionClose: - events = append(events, rec) - continue - } - if _, newer := r.latest[rec.SessionID]; newer { - continue - } - if _, live := r.sessions[rec.SessionID]; live { - r.latest[rec.SessionID] = rec - } - } - r.events = append(events, r.events...) - if over := len(r.events) - maxPendingEvents; over > 0 { - slog.Warn("relay: usage backlog full; dropping the oldest session events", "dropped", over) - r.events = r.events[over:] - } -} - -func (r *usageReporter) send(ctx context.Context, batch []usage.Record) error { - body, err := json.Marshal(batch) - if err != nil { - return fmt.Errorf("encode usage: %w", err) - } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.url, bytes.NewReader(body)) - if err != nil { - return err - } - if r.token != "" { - req.Header.Set("Authorization", "Bearer "+r.token) - } - req.Header.Set("Content-Type", "application/json") - // Signed at each send: a batch sent again after a failure carries a - // fresh timestamp. - if err := r.signer.Sign(req, body); err != nil { - return fmt.Errorf("usage report: %w", err) - } - resp, err := r.client.Do(req) - if err != nil { - return err - } - defer func() { _ = resp.Body.Close() }() // not actionable: drained below - _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, maxBody)) // not actionable: drained to reuse the connection - r.refusals.Observe(r.url, resp) - switch { - case resp.StatusCode >= 200 && resp.StatusCode < 300: - return nil - case resp.StatusCode == http.StatusBadRequest || resp.StatusCode == http.StatusUnprocessableEntity: - return fmt.Errorf("%w: status %d", errMalformed, resp.StatusCode) - default: - return fmt.Errorf("status %d", resp.StatusCode) - } -} - // since returns what b moved beyond base, each total at least zero. func (b Bytes) since(base Bytes) Bytes { return Bytes{Sent: b.Sent - min(b.Sent, base.Sent), Received: b.Received - min(b.Received, base.Received)} diff --git a/internal/auth/usage_viewers_test.go b/internal/auth/usage_viewers_test.go index 3f88402b..b2266114 100644 --- a/internal/auth/usage_viewers_test.go +++ b/internal/auth/usage_viewers_test.go @@ -36,7 +36,7 @@ func TestUsageReporter_ViewersAreTotalled(t *testing.T) { for _, rec := range first { byKID[rec.KID] = rec assert.Equal(t, usage.Viewers, rec.Type) - assert.Equal(t, r.runID, rec.Run) + assert.Equal(t, r.meter.RunID(), rec.Run) assert.Empty(t, rec.SessionID) assert.NoError(t, rec.Validate()) } @@ -95,11 +95,6 @@ func TestUsageReporter_PublishersStayPerSession(t *testing.T) { assert.ElementsMatch(t, []string{"session_open p1", "session_open p2", "usage p1"}, got) } -func TestUsageReporter_EachRunHasItsOwnTotals(t *testing.T) { - a, b := newTestUsage(t, &fakeUsageSink{}), newTestUsage(t, &fakeUsageSink{}) - assert.NotEqual(t, a.runID, b.runID, "a restarted relay starts new totals rather than lowering old ones") -} - // publishing turns a viewer into a publisher once: a session it doesn't know, // or one that publishes already, gets no second open. func TestUsageReporter_Publishing(t *testing.T) { diff --git a/internal/auth/verifier_test.go b/internal/auth/verifier_test.go index d02d3f32..b16478d4 100644 --- a/internal/auth/verifier_test.go +++ b/internal/auth/verifier_test.go @@ -6,7 +6,6 @@ import ( "encoding/base64" "encoding/json" "errors" - "fmt" "net/http" "net/http/httptest" "net/url" @@ -430,61 +429,8 @@ func TestUsageReporter(t *testing.T) { assert.Equal(t, int64(20), retry[1].Metrics["gateway.ingress_bytes"], "the newer report wins") }) - for _, status := range []int{http.StatusBadRequest, http.StatusUnprocessableEntity} { - t.Run(fmt.Sprintf("a malformed batch (%d) is dropped", status), func(t *testing.T) { - sink := &fakeUsageSink{statuses: []int{status, http.StatusOK}} - r := newTestUsage(t, sink) - r.open("s1", testUsageSession) - require.NoError(t, r.flush(context.Background())) - require.NoError(t, r.flush(context.Background())) - assert.Len(t, sink.received(), 1) - }) - } - - // A rotated token (401, 403), a body over the receiver's limit (413), a - // timeout (408) or throttling (429) lose nothing: the records are kept. - for _, status := range []int{http.StatusUnauthorized, http.StatusForbidden, http.StatusRequestTimeout, - http.StatusRequestEntityTooLarge, http.StatusTooManyRequests, http.StatusBadGateway} { - t.Run(fmt.Sprintf("%d keeps the records", status), func(t *testing.T) { - sink := &fakeUsageSink{statuses: []int{status, http.StatusOK}} - r := newTestUsage(t, sink) - r.open("s1", testUsageSession) - assert.Error(t, r.flush(context.Background())) - require.NoError(t, r.flush(context.Background())) - batches := sink.received() - require.Len(t, batches, 2) - assert.Equal(t, batches[0], batches[1], "the same records again") - }) - } - - t.Run("a backlog goes out in batches", func(t *testing.T) { - sink := &fakeUsageSink{} - r := newTestUsage(t, sink) - for i := range 1200 { - r.open(fmt.Sprintf("s%d", i), testUsageSession) - } - require.NoError(t, r.flush(context.Background())) - var sizes []int - for _, b := range sink.received() { - sizes = append(sizes, len(b)) - } - assert.Equal(t, []int{500, 500, 200}, sizes) - }) - - t.Run("a failure mid-backlog resends only what wasn't sent", func(t *testing.T) { - sink := &fakeUsageSink{statuses: []int{http.StatusOK, http.StatusServiceUnavailable, http.StatusOK}} - r := newTestUsage(t, sink) - for i := range 700 { - r.open(fmt.Sprintf("s%d", i), testUsageSession) - } - assert.Error(t, r.flush(context.Background())) - require.NoError(t, r.flush(context.Background())) - var sizes []int - for _, b := range sink.received() { - sizes = append(sizes, len(b)) - } - assert.Equal(t, []int{500, 200, 200}, sizes, "the first 500 went through; the next 200 are resent once") - }) + // Batching, retries and what each answer from the receiver does are + // package meter's, tested there. t.Run("an unknown session's end and usage are ignored", func(t *testing.T) { sink := &fakeUsageSink{} @@ -1031,7 +977,8 @@ func TestUsageReporter_OpenAfterClose(t *testing.T) { r.publishing("s1") assert.Empty(t, r.sessions, "an open after the close is ignored") - assert.Empty(t, r.events) + require.NoError(t, r.flush(context.Background())) + assert.Empty(t, sink.received(), "no open is sent") now = now.Add(sessionGrace + time.Second) r.forgetStale() diff --git a/internal/meter/meter.go b/internal/meter/meter.go new file mode 100644 index 00000000..b41576b5 --- /dev/null +++ b/internal/meter/meter.go @@ -0,0 +1,399 @@ +// Package meter reports usage records (package usage) to a usage URL +// (QUMO_USAGE_URL), for any qumo server that meters what its tenants use. +// +// A server opens a session for each thing it meters under a tenant's key, +// such as a relay session or an ingested stream, reports its cumulative +// metrics while it lasts, and closes it with its final totals. Usage that +// isn't worth a record per session, such as a relay's viewers, goes in one +// running total per key for the run of the server. Every record is +// cumulative, so a receiver can take a resent one without counting it twice. +// +// Records wait in memory between sends, every 10 s, and through an outage; a +// session's usage is coalesced to its latest report. +package meter + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "maps" + "net" + "net/http" + "net/url" + "sync" + "time" + + "github.com/qumo-dev/qumo/internal/reqsign" + "github.com/qumo-dev/qumo/usage" +) + +const ( + // flushInterval is how often pending records are sent. + flushInterval = 10 * time.Second + // sendTimeout bounds one send, and the last send at shutdown. + sendTimeout = 10 * time.Second + // maxBatchRecords bounds one POST, so a backlog after an outage goes out + // in pieces a receiver's body limit accepts. + maxBatchRecords = 500 + // maxPendingEvents caps the opens and closes waiting while the receiver + // can't be reached; the oldest are dropped past it. Usage needs no cap: + // it is one cumulative record per live session. + maxPendingEvents = 50_000 + // maxBody bounds how much of a reply's body is drained. + maxBody = 64 << 10 +) + +// Config is where a Reporter sends its records. +type Config struct { + // Name says who reports, in the log: "relay", "ingest". + Name string + // URL receives the records: https, or http to a loopback host. + URL string + // Token, when set, is sent as a bearer token. + Token string + // Signer, when set, signs each request with the server's identity. + Signer *reqsign.Signer +} + +// total is one key's running total for this run, and what the receiver has +// acknowledged of it. +type total struct { + metrics map[string]int64 + reported map[string]int64 + sent bool // reported at least once +} + +// A Reporter sends usage records to a usage URL. Its methods are safe for +// concurrent use. +type Reporter struct { + name string + url string + token string + signer *reqsign.Signer // nil: requests are not signed + client *http.Client + // refusals logs the receiver's 401s, the first of a run only. + refusals reqsign.Refusals + now func() time.Time + // runID names this run of the server in its totals: a restarted server + // starts new totals rather than lowering old ones. + runID string + + mu sync.Mutex + live map[string]bool // sessions opened, and not closed or forgotten + events []usage.Record // opens and closes, in order + latest map[string]usage.Record // latest cumulative report per session + totals map[string]*total // by kid + failing bool // the last send failed +} + +// New returns a Reporter sending to cfg.URL. Run sends its records. +func New(cfg Config) (*Reporter, error) { + if err := checkURL(cfg.URL); err != nil { + return nil, err + } + return &Reporter{ + name: cfg.Name, + url: cfg.URL, + token: cfg.Token, + signer: cfg.Signer, + client: reqsign.NoRedirectClient(sendTimeout), + now: time.Now, + runID: rand.Text(), + live: map[string]bool{}, + latest: map[string]usage.Record{}, + totals: map[string]*total{}, + }, nil +} + +// checkURL accepts https, and http to a loopback host only. +func checkURL(rawURL string) error { + u, err := url.Parse(rawURL) + if err != nil { + return err + } + switch u.Scheme { + case "https": + case "http": + host := u.Hostname() + if ip := net.ParseIP(host); host != "localhost" && (ip == nil || !ip.IsLoopback()) { + return fmt.Errorf("http is allowed only for a loopback host, got %q", host) + } + default: + return fmt.Errorf("want an https URL, got scheme %q", u.Scheme) + } + return nil +} + +// RunID returns the run named in r's totals. +func (r *Reporter) RunID() string { return r.runID } + +// Open records that session id began under key kid: a session_open. +func (r *Reporter) Open(id, kid string) { + r.mu.Lock() + defer r.mu.Unlock() + r.live[id] = true + r.appendEvent(r.record(usage.SessionOpen, id, kid)) +} + +// Report records the cumulative metrics of session id so far: a usage +// record, replacing any not yet sent. A session not open is ignored. +func (r *Reporter) Report(id, kid string, metrics map[string]int64) { + r.mu.Lock() + defer r.mu.Unlock() + if !r.live[id] { + return + } + rec := r.record(usage.Usage, id, kid) + rec.Metrics = maps.Clone(metrics) + r.latest[id] = rec +} + +// Close records the end of session id with its final totals and the reason +// it ended: a session_close, which supersedes its pending usage. A session +// not open is ignored. +func (r *Reporter) Close(id, kid string, metrics map[string]int64, reason string) { + r.mu.Lock() + defer r.mu.Unlock() + if !r.live[id] { + return + } + delete(r.live, id) + delete(r.latest, id) // the close carries totals at least as large + rec := r.record(usage.SessionClose, id, kid) + rec.Metrics = maps.Clone(metrics) + rec.Reason = reason + r.appendEvent(rec) +} + +// Forget drops session id without a close, for one whose end never came. Its +// usage not yet sent still goes, but is not sent again after a failure. +func (r *Reporter) Forget(id string) { + r.mu.Lock() + defer r.mu.Unlock() + delete(r.live, id) +} + +// Total records the running total of key kid for this run: a total record +// (usage.Viewers), sent whenever it differs from what the receiver last +// acknowledged. +func (r *Reporter) Total(kid string, metrics map[string]int64) { + r.mu.Lock() + defer r.mu.Unlock() + t := r.totals[kid] + if t == nil { + t = &total{} + r.totals[kid] = t + } + t.metrics = maps.Clone(metrics) +} + +func (r *Reporter) record(typ usage.Type, id, kid string) usage.Record { + return usage.Record{Type: typ, SessionID: id, KID: kid, TS: r.timestamp()} +} + +// timestamp is a record's ts: now, to the second. +func (r *Reporter) timestamp() time.Time { + return r.now().UTC().Truncate(time.Second) +} + +// appendEvent queues an open or close, dropping the oldest past the cap. +// The caller holds mu. +func (r *Reporter) appendEvent(rec usage.Record) { + r.events = append(r.events, rec) + r.capEvents() +} + +// capEvents drops the oldest events past maxPendingEvents. The caller holds +// mu. +func (r *Reporter) capEvents() { + if over := len(r.events) - maxPendingEvents; over > 0 { + slog.Warn(r.name+": usage backlog full; dropping the oldest session events", "dropped", over) + r.events = r.events[over:] + } +} + +// Run sends pending records every 10 s until ctx ends, then once more. +func (r *Reporter) Run(ctx context.Context) { + ticker := time.NewTicker(flushInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + last, cancel := context.WithTimeout(context.WithoutCancel(ctx), sendTimeout) + defer cancel() + if err := r.Flush(last); err != nil { + slog.Error(r.name+": last usage send failed; records lost", "error", err) + } + return + case <-ticker.C: + r.logOutcome(ctx, r.Flush(ctx)) + } + } +} + +// logOutcome logs a send's first failure as a warning, repeats quietly, and +// the recovery as info: during an outage the server retries every few +// seconds, and one line says so. +func (r *Reporter) logOutcome(ctx context.Context, err error) { + r.mu.Lock() + wasFailing := r.failing + r.failing = err != nil + r.mu.Unlock() + switch { + case err != nil && ctx.Err() != nil: + case err != nil && !wasFailing: + slog.Warn(r.name+": usage send failed; keeping the records and retrying", "error", err) + case err != nil: + slog.Debug(r.name+": usage send still failing", "error", err) + case wasFailing: + slog.Info(r.name + ": usage send recovered") + } +} + +// errMalformed is a 400 or 422 answer to a batch: the receiver can't read +// it, so resending it can't succeed. +var errMalformed = errors.New("usage batch malformed") + +// Flush sends every pending record: opens, then per-session usage, then the +// totals that changed, then closes, in batches of at most 500. A batch the +// receiver can't read (400, 422) is dropped, so it can't block later +// records. Any other failure, a 401 or 413 included, stops the flush and puts +// the unsent records back, unless newer ones replaced them, to try again +// next time. +func (r *Reporter) Flush(ctx context.Context) error { + r.mu.Lock() + events, latest := r.events, r.latest + r.events, r.latest = nil, map[string]usage.Record{} + // A total is sent when it has changed since the receiver last + // acknowledged it. It stays pending until then, so it needs no requeue. + sending := map[string]map[string]int64{} // by kid + var totals []usage.Record + for kid, t := range r.totals { + if t.sent && maps.Equal(t.metrics, t.reported) { + continue + } + totals = append(totals, usage.Record{ + Type: usage.Viewers, Run: r.runID, KID: kid, Metrics: maps.Clone(t.metrics), TS: r.timestamp(), + }) + sending[kid] = t.metrics + } + r.mu.Unlock() + if len(events) == 0 && len(latest) == 0 && len(totals) == 0 { + return nil + } + + all := make([]usage.Record, 0, len(events)+len(latest)+len(totals)) + for _, e := range events { + if e.Type == usage.SessionOpen { + all = append(all, e) + } + } + for _, u := range latest { + all = append(all, u) + } + all = append(all, totals...) + for _, e := range events { + if e.Type == usage.SessionClose { + all = append(all, e) + } + } + + for start := 0; start < len(all); start += maxBatchRecords { + batch := all[start:min(start+maxBatchRecords, len(all))] + err := r.send(ctx, batch) + switch { + case err == nil: + r.acknowledge(batch, sending) + case errors.Is(err, errMalformed): + slog.Error(r.name+": usage batch refused as malformed; dropped", "records", len(batch), "error", err) + // Not resent as it is; a total goes again when it changes. + r.acknowledge(batch, sending) + default: + r.requeue(all[start:]) + return err + } + } + return nil +} + +// acknowledge records the totals in a sent batch, by kid in sending, as +// reported. +func (r *Reporter) acknowledge(batch []usage.Record, sending map[string]map[string]int64) { + r.mu.Lock() + defer r.mu.Unlock() + for _, rec := range batch { + if rec.Type != usage.Viewers { + continue + } + if t := r.totals[rec.KID]; t != nil { + t.reported, t.sent = sending[rec.KID], true + } + } +} + +// requeue puts unsent records back before anything recorded since: opens +// and closes in order, and a session's usage record only if the session is +// still live and has no newer one. Totals aren't put back: they stay pending +// until acknowledged. +func (r *Reporter) requeue(unsent []usage.Record) { + r.mu.Lock() + defer r.mu.Unlock() + var events []usage.Record + for _, rec := range unsent { + switch rec.Type { + case usage.Viewers: + continue + case usage.SessionOpen, usage.SessionClose: + events = append(events, rec) + continue + } + if _, newer := r.latest[rec.SessionID]; newer { + continue + } + if r.live[rec.SessionID] { + r.latest[rec.SessionID] = rec + } + } + r.events = append(events, r.events...) + r.capEvents() +} + +func (r *Reporter) send(ctx context.Context, batch []usage.Record) error { + body, err := json.Marshal(batch) + if err != nil { + return fmt.Errorf("encode usage: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.url, bytes.NewReader(body)) + if err != nil { + return err + } + if r.token != "" { + req.Header.Set("Authorization", "Bearer "+r.token) + } + req.Header.Set("Content-Type", "application/json") + // Signed at each send: a batch sent again after a failure carries a + // fresh timestamp. + if err := r.signer.Sign(req, body); err != nil { + return fmt.Errorf("usage report: %w", err) + } + resp, err := r.client.Do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() // not actionable: drained below + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, maxBody)) // not actionable: drained to reuse the connection + r.refusals.Observe(r.url, resp) + switch { + case resp.StatusCode >= 200 && resp.StatusCode < 300: + return nil + case resp.StatusCode == http.StatusBadRequest || resp.StatusCode == http.StatusUnprocessableEntity: + return fmt.Errorf("%w: status %d", errMalformed, resp.StatusCode) + default: + return fmt.Errorf("status %d", resp.StatusCode) + } +} diff --git a/internal/meter/meter_test.go b/internal/meter/meter_test.go new file mode 100644 index 00000000..4100b766 --- /dev/null +++ b/internal/meter/meter_test.go @@ -0,0 +1,305 @@ +package meter + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "sync" + "testing" + "testing/synctest" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/qumo-dev/qumo/usage" +) + +// fakeSink records the batches POSTed to it and answers from a status queue: +// in order, the last repeating; empty means 200. +type fakeSink struct { + mu sync.Mutex + statuses []int + batches [][]usage.Record + gotAuth string +} + +func (f *fakeSink) ServeHTTP(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + f.gotAuth = r.Header.Get("Authorization") + var batch []usage.Record + if err := json.NewDecoder(r.Body).Decode(&batch); err == nil { + f.batches = append(f.batches, batch) + } + status := http.StatusOK + if len(f.statuses) > 0 { + status = f.statuses[0] + if len(f.statuses) > 1 { + f.statuses = f.statuses[1:] + } + } + w.WriteHeader(status) +} + +func (f *fakeSink) received() [][]usage.Record { + f.mu.Lock() + defer f.mu.Unlock() + return append([][]usage.Record(nil), f.batches...) +} + +func newTestReporter(t *testing.T, sink *fakeSink) *Reporter { + t.Helper() + srv := httptest.NewServer(sink) + t.Cleanup(srv.Close) + r, err := New(Config{Name: "test", URL: srv.URL, Token: "t"}) + require.NoError(t, err) + return r +} + +// summary is each record of batch as "type session_id" or "type kid". +func summary(batch []usage.Record) []string { + var out []string + for _, rec := range batch { + id := rec.SessionID + if rec.Type == usage.Viewers { + id = rec.KID + } + out = append(out, string(rec.Type)+" "+id) + } + return out +} + +func bytesMetrics(in, out int64) map[string]int64 { + return map[string]int64{usage.IngressBytes: in, usage.EgressBytes: out} +} + +func TestNew(t *testing.T) { + for rawURL, wantErr := range map[string]bool{ + "https://usage.example/v1": false, + "http://127.0.0.1:9000/v1": false, + "http://localhost/v1": false, + "http://usage.example/v1": true, + "ftp://usage.example/v1": true, + "https://usage.example/%zz": true, + } { + _, err := New(Config{URL: rawURL}) + assert.Equal(t, wantErr, err != nil, rawURL) + } +} + +// A session's records go out in order: opens, usage, totals, closes, each +// valid, with the bearer token. +func TestReporter_Records(t *testing.T) { + sink := &fakeSink{} + r := newTestReporter(t, sink) + r.Open("s1", "k1") + r.Open("s2", "k1") + r.Report("s1", "k1", bytesMetrics(10, 1)) + r.Report("s1", "k1", bytesMetrics(20, 2)) + r.Total("k2", bytesMetrics(0, 7)) + r.Close("s2", "k1", bytesMetrics(5, 5), usage.ReasonClosed) + + require.NoError(t, r.Flush(context.Background())) + + batches := sink.received() + require.Len(t, batches, 1) + assert.Equal(t, []string{"session_open s1", "session_open s2", "usage s1", "viewers k2", "session_close s2"}, summary(batches[0])) + for _, rec := range batches[0] { + assert.NoError(t, rec.Validate()) + } + assert.Equal(t, bytesMetrics(20, 2), batches[0][2].Metrics, "usage is coalesced to the latest report") + assert.Equal(t, r.RunID(), batches[0][3].Run) + assert.Equal(t, usage.ReasonClosed, batches[0][4].Reason) + assert.Equal(t, "Bearer t", sink.gotAuth) +} + +func TestReporter_CloseSupersedesUsage(t *testing.T) { + sink := &fakeSink{} + r := newTestReporter(t, sink) + r.Open("s1", "k1") + r.Report("s1", "k1", bytesMetrics(10, 1)) + r.Close("s1", "k1", bytesMetrics(30, 3), usage.ReasonClosed) + + require.NoError(t, r.Flush(context.Background())) + + assert.Equal(t, []string{"session_open s1", "session_close s1"}, summary(sink.received()[0])) +} + +// Usage and a close for a session not open, closed already or forgotten are +// ignored. +func TestReporter_NotOpen(t *testing.T) { + sink := &fakeSink{} + r := newTestReporter(t, sink) + r.Report("nope", "k1", bytesMetrics(1, 1)) + r.Close("nope", "k1", bytesMetrics(1, 1), usage.ReasonClosed) + r.Open("s1", "k1") + r.Close("s1", "k1", bytesMetrics(1, 1), usage.ReasonClosed) + r.Close("s1", "k1", bytesMetrics(2, 2), usage.ReasonClosed) + r.Report("s1", "k1", bytesMetrics(3, 3)) + r.Open("s2", "k1") + r.Forget("s2") + r.Report("s2", "k1", bytesMetrics(3, 3)) + + require.NoError(t, r.Flush(context.Background())) + + assert.Equal(t, []string{"session_open s1", "session_open s2", "session_close s1"}, summary(sink.received()[0])) +} + +func TestReporter_FailedSendIsRetriedWithNewerUsage(t *testing.T) { + sink := &fakeSink{statuses: []int{http.StatusBadGateway, http.StatusOK}} + r := newTestReporter(t, sink) + r.Open("s1", "k1") + r.Report("s1", "k1", bytesMetrics(10, 0)) + assert.Error(t, r.Flush(context.Background())) + r.Report("s1", "k1", bytesMetrics(20, 0)) + + require.NoError(t, r.Flush(context.Background())) + + retry := sink.received()[1] + assert.Equal(t, []string{"session_open s1", "usage s1"}, summary(retry), "the open is resent") + assert.Equal(t, int64(20), retry[1].Metrics[usage.IngressBytes], "the newer report wins") +} + +// A forgotten session's usage goes once, but not again after a failure. +func TestReporter_ForgottenUsageIsNotResent(t *testing.T) { + sink := &fakeSink{statuses: []int{http.StatusOK, http.StatusBadGateway, http.StatusOK}} + r := newTestReporter(t, sink) + r.Open("s1", "k1") + require.NoError(t, r.Flush(context.Background())) + r.Report("s1", "k1", bytesMetrics(10, 0)) + r.Forget("s1") + + assert.Error(t, r.Flush(context.Background())) + require.NoError(t, r.Flush(context.Background())) + + assert.Len(t, sink.received(), 2, "the open, then the usage once; nothing to resend") + assert.Equal(t, []string{"usage s1"}, summary(sink.received()[1])) +} + +func TestReporter_MalformedBatchIsDropped(t *testing.T) { + for _, status := range []int{http.StatusBadRequest, http.StatusUnprocessableEntity} { + t.Run(fmt.Sprint(status), func(t *testing.T) { + sink := &fakeSink{statuses: []int{status, http.StatusOK}} + r := newTestReporter(t, sink) + r.Open("s1", "k1") + require.NoError(t, r.Flush(context.Background())) + require.NoError(t, r.Flush(context.Background())) + assert.Len(t, sink.received(), 1) + }) + } +} + +// A rotated token (401, 403), a body over the receiver's limit (413), a +// timeout (408) or throttling (429) lose nothing: the records are kept. +func TestReporter_FailureKeepsTheRecords(t *testing.T) { + for _, status := range []int{http.StatusUnauthorized, http.StatusForbidden, http.StatusRequestTimeout, + http.StatusRequestEntityTooLarge, http.StatusTooManyRequests, http.StatusBadGateway} { + t.Run(fmt.Sprint(status), func(t *testing.T) { + sink := &fakeSink{statuses: []int{status, http.StatusOK}} + r := newTestReporter(t, sink) + r.Open("s1", "k1") + assert.Error(t, r.Flush(context.Background())) + require.NoError(t, r.Flush(context.Background())) + batches := sink.received() + require.Len(t, batches, 2) + assert.Equal(t, batches[0], batches[1], "the same records again") + }) + } +} + +func TestReporter_Batches(t *testing.T) { + t.Run("a backlog goes out in batches", func(t *testing.T) { + sink := &fakeSink{} + r := newTestReporter(t, sink) + for i := range 1200 { + r.Open(fmt.Sprintf("s%d", i), "k1") + } + require.NoError(t, r.Flush(context.Background())) + var sizes []int + for _, b := range sink.received() { + sizes = append(sizes, len(b)) + } + assert.Equal(t, []int{500, 500, 200}, sizes) + }) + + t.Run("a failure mid-backlog resends only what wasn't sent", func(t *testing.T) { + sink := &fakeSink{statuses: []int{http.StatusOK, http.StatusServiceUnavailable, http.StatusOK}} + r := newTestReporter(t, sink) + for i := range 700 { + r.Open(fmt.Sprintf("s%d", i), "k1") + } + assert.Error(t, r.Flush(context.Background())) + require.NoError(t, r.Flush(context.Background())) + var sizes []int + for _, b := range sink.received() { + sizes = append(sizes, len(b)) + } + assert.Equal(t, []int{500, 200, 200}, sizes, "the first 500 went through; the next 200 are resent once") + }) +} + +func TestReporter_Totals(t *testing.T) { + sink := &fakeSink{statuses: []int{http.StatusOK, http.StatusServiceUnavailable, http.StatusOK}} + r := newTestReporter(t, sink) + r.Total("k1", bytesMetrics(0, 100)) + require.NoError(t, r.Flush(context.Background())) + + r.Total("k1", bytesMetrics(0, 100)) + require.NoError(t, r.Flush(context.Background())) + assert.Len(t, sink.received(), 1, "an unchanged total isn't sent again") + + r.Total("k1", bytesMetrics(0, 150)) + assert.Error(t, r.Flush(context.Background())) + r.Total("k1", bytesMetrics(0, 160)) + require.NoError(t, r.Flush(context.Background())) + + batches := sink.received() + require.Len(t, batches, 3) + require.Len(t, batches[2], 1, "one record, not the old total and the new") + assert.Equal(t, int64(160), batches[2][0].Metrics[usage.EgressBytes]) +} + +func TestReporter_EachRunHasItsOwnTotals(t *testing.T) { + a, b := newTestReporter(t, &fakeSink{}), newTestReporter(t, &fakeSink{}) + assert.NotEqual(t, a.RunID(), b.RunID(), "a restarted server starts new totals rather than lowering old ones") +} + +// Run sends every flushInterval, and once more when ctx ends. +func TestReporter_Run(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var mu sync.Mutex + var sent []time.Time + r, err := New(Config{Name: "test", URL: "http://127.0.0.1/usage"}) + require.NoError(t, err) + r.client = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + mu.Lock() + defer mu.Unlock() + sent = append(sent, time.Now()) + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody}, nil + })} + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan struct{}) + start := time.Now() + go func() { + defer close(done) + r.Run(ctx) + }() + + r.Open("s1", "k1") + time.Sleep(flushInterval + time.Second) + r.Close("s1", "k1", bytesMetrics(1, 1), usage.ReasonClosed) + time.Sleep(time.Second) + cancel() + <-done + + assert.Equal(t, []time.Time{start.Add(flushInterval), start.Add(flushInterval + 2*time.Second)}, sent, + "at the tick, then the close at shutdown") + }) +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } From 99c79429532fe996b2a097891cd536483feda6a6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 19:26:21 +0000 Subject: [PATCH 2/3] docs(changelog): keep the meter entry under Unreleased, not v0.21.261118 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KPWMJL7PH3rPhK5UTGavaB --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2160476..31c0c5c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- **Usage reporting is a package of its own, for every server to use (`internal/meter`).** What sends usage records (`usage.Record`) to `QUMO_USAGE_URL` is no longer the relay's: `meter.Reporter` takes a server's sessions (`Open`, `Report` with cumulative metrics, `Close`) and running totals per key (`Total`), and keeps them in memory, coalesces each session's usage, batches by 500, retries through an outage, signs the requests when given a signer and sends what is left at shutdown, as the relay did. The relay reports through it, with what it decides itself (a publisher by itself, viewers in one total per key) kept in `internal/auth`. The records the relay sends are unchanged. + ## [v0.21.261118] - 2026-11-18 ### Security @@ -17,10 +21,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **A relay signs its key-set and usage requests, and `qumo dns` its feed reads, with its peer identity (`internal/reqsign`, #601).** With `PEER_CERT_FILE` and `PEER_KEY_FILE` (an ECDSA key; `qumo dns` takes the same two settings), each request carries `X-Relay-Certificate`, `X-Relay-Timestamp` and `X-Relay-Signature`: the key's ECDSA signature (ASN.1, base64) over SHA-256 of `qumo-relay-request`, the method, the request URI (path and query), the timestamp and the hex SHA-256 of the body, one per line, signed again on every retry. The receiver then knows which relay is calling from a key only that relay holds, and a captured request can't be changed or used for long, where a bearer token copied from a node could be reused until it is replaced. The bearer token (`QUMO_RELAY_TOKEN`, `QUMO_DNS_FEED_TOKEN`) is still sent beside it. A 401 is logged with the server's `Date` beside the local time. Funnel, HLS and ingest, which have no peer identity, are unchanged. -### Changed - -- **Usage reporting is a package of its own, for every server to use (`internal/meter`).** What sends usage records (`usage.Record`) to `QUMO_USAGE_URL` is no longer the relay's: `meter.Reporter` takes a server's sessions (`Open`, `Report` with cumulative metrics, `Close`) and running totals per key (`Total`), and keeps them in memory, coalesces each session's usage, batches by 500, retries through an outage, signs the requests when given a signer and sends what is left at shutdown, as the relay did. The relay reports through it, with what it decides itself (a publisher by itself, viewers in one total per key) kept in `internal/auth`. The records the relay sends are unchanged. - ## [v0.20.261114] - 2026-11-14 ### Added From 2dda76d854775f1ef6ec061f0238fe2e3f4756e9 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 11:03:21 +0000 Subject: [PATCH 3/3] refactor(meter): one control-plane URL check; drop hooks nothing uses - The check that a key-set or usage URL is https, or http only to a loopback host, was in internal/auth and copied into internal/meter. It is now reqsign.CheckURL, beside the other rules for requests to the control plane (NoRedirectClient, Refusals), so the two can't drift. - meter.Reporter.now was never overridden; it uses time.Now. - usageReporter.flush existed only for tests; they call meter.Flush. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KPWMJL7PH3rPhK5UTGavaB --- internal/auth/auth.go | 22 ---------------------- internal/auth/auth_test.go | 24 ------------------------ internal/auth/keysource.go | 2 +- internal/auth/signed_requests_test.go | 4 ++-- internal/auth/usage.go | 3 --- internal/auth/usage_viewers_test.go | 14 +++++++------- internal/auth/verifier_test.go | 20 ++++++++++---------- internal/meter/meter.go | 27 ++------------------------- internal/meter/meter_test.go | 16 +++++----------- internal/reqsign/reqsign.go | 23 +++++++++++++++++++++++ internal/reqsign/reqsign_test.go | 23 +++++++++++++++++++++++ 11 files changed, 73 insertions(+), 105 deletions(-) diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 7132fb05..010991cb 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -12,9 +12,7 @@ package auth import ( "errors" "fmt" - "net" "net/http" - "net/url" ) // maxBody bounds how much of a reply's body is drained. @@ -60,26 +58,6 @@ func RefusalStatus(err error) int { return http.StatusServiceUnavailable } -// checkURL accepts an https URL, or http only on a loopback host: the bearer -// token the relay sends to it must not cross a network in the clear. -func checkURL(rawURL string) error { - u, err := url.Parse(rawURL) - if err != nil { - return err - } - switch u.Scheme { - case "https": - case "http": - host := u.Hostname() - if ip := net.ParseIP(host); host != "localhost" && (ip == nil || !ip.IsLoopback()) { - return fmt.Errorf("http is allowed only for a loopback host, got %q", host) - } - default: - return fmt.Errorf("want an https URL, got scheme %q", u.Scheme) - } - return nil -} - // Request is one session event: what the relay asks its Authorize about at // connect and revalidate, and reports to its End. type Request struct { diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go index b3c80145..08bf8e0c 100644 --- a/internal/auth/auth_test.go +++ b/internal/auth/auth_test.go @@ -9,30 +9,6 @@ import ( "github.com/stretchr/testify/assert" ) -func TestCheckURL(t *testing.T) { - tests := map[string]struct { - url string - wantErrText string - }{ - "https": {url: "https://keys.example.com/v1/set"}, - "loopback http": {url: "http://127.0.0.1:4440/"}, - "localhost http": {url: "http://localhost:4440/"}, - "http off loopback": {url: "http://keys.example.com/", wantErrText: "loopback"}, - "another scheme": {url: "ftp://keys.example.com/", wantErrText: "https"}, - } - for name, tt := range tests { - t.Run(name, func(t *testing.T) { - err := checkURL(tt.url) - - if tt.wantErrText != "" { - assert.ErrorContains(t, err, tt.wantErrText) - return - } - assert.NoError(t, err) - }) - } -} - func TestRefusalStatus(t *testing.T) { tests := map[string]struct { err error diff --git a/internal/auth/keysource.go b/internal/auth/keysource.go index 421de59b..a36e9a08 100644 --- a/internal/auth/keysource.go +++ b/internal/auth/keysource.go @@ -306,7 +306,7 @@ func (u *urlKeySource) touchCache(now time.Time) { } func newURLKeySource(rawURL, bearer string, signer *reqsign.Signer) (*urlKeySource, error) { - if err := checkURL(rawURL); err != nil { + if err := reqsign.CheckURL(rawURL); err != nil { return nil, err } return &urlKeySource{url: rawURL, token: bearer, signer: signer, client: reqsign.NoRedirectClient(keysFetchTimeout)}, nil diff --git a/internal/auth/signed_requests_test.go b/internal/auth/signed_requests_test.go index 0fd0b33c..276e32a7 100644 --- a/internal/auth/signed_requests_test.go +++ b/internal/auth/signed_requests_test.go @@ -86,8 +86,8 @@ func TestVerifier_SignsItsRequests(t *testing.T) { require.NoError(t, v.source.refresh(t.Context(), &store, time.Now())) v.usage.open("s1", testUsageSession) - assert.Error(t, v.usage.flush(t.Context())) - require.NoError(t, v.usage.flush(t.Context())) + assert.Error(t, v.usage.meter.Flush(t.Context())) + require.NoError(t, v.usage.meter.Flush(t.Context())) require.Len(t, keyReqs.all(), 1) assertSigned(t, keyReqs.all()[0], "relay-token") diff --git a/internal/auth/usage.go b/internal/auth/usage.go index 0d9d461a..3832ff0d 100644 --- a/internal/auth/usage.go +++ b/internal/auth/usage.go @@ -193,9 +193,6 @@ func (r *usageReporter) run(ctx context.Context) { wg.Wait() } -// flush sends every pending record now. -func (r *usageReporter) flush(ctx context.Context) error { return r.meter.Flush(ctx) } - // forgetStale drops sessions whose end never came: those not heard of for // sessionGrace. func (r *usageReporter) forgetStale() { diff --git a/internal/auth/usage_viewers_test.go b/internal/auth/usage_viewers_test.go index b2266114..79ee16e7 100644 --- a/internal/auth/usage_viewers_test.go +++ b/internal/auth/usage_viewers_test.go @@ -28,7 +28,7 @@ func TestUsageReporter_ViewersAreTotalled(t *testing.T) { r.revalidated("v1", Bytes{Sent: 100, Received: 10}, true) r.revalidated("v2", Bytes{Sent: 200, Received: 20}, true) r.revalidated("other", Bytes{Sent: 7}, true) - require.NoError(t, r.flush(context.Background())) + require.NoError(t, r.meter.Flush(context.Background())) first := sink.received()[0] require.Len(t, first, 2, "one record per key, not per viewer") @@ -44,7 +44,7 @@ func TestUsageReporter_ViewersAreTotalled(t *testing.T) { assert.Equal(t, int64(7), byKID["k2"].Metrics["gateway.egress_bytes"]) t.Run("an unchanged total isn't sent again", func(t *testing.T) { - require.NoError(t, r.flush(context.Background())) + require.NoError(t, r.meter.Flush(context.Background())) assert.Len(t, sink.received(), 1) }) @@ -54,7 +54,7 @@ func TestUsageReporter_ViewersAreTotalled(t *testing.T) { assert.True(t, r.close("v2", Bytes{Sent: 260, Received: 25}, "closed")) // +60, +5 r.revalidated("v2", Bytes{Sent: 999}, true) // after its close: ignored assert.True(t, r.close("v3", Bytes{}, "closed")) // moved nothing - require.NoError(t, r.flush(context.Background())) + require.NoError(t, r.meter.Flush(context.Background())) batches := sink.received() require.Len(t, batches, 2) @@ -69,9 +69,9 @@ func TestUsageReporter_ViewerTotalSurvivesAFailedSend(t *testing.T) { r.open("v1", viewerSession("k1")) r.revalidated("v1", Bytes{Sent: 100}, true) - assert.Error(t, r.flush(context.Background())) + assert.Error(t, r.meter.Flush(context.Background())) r.revalidated("v1", Bytes{Sent: 160}, true) - require.NoError(t, r.flush(context.Background())) + require.NoError(t, r.meter.Flush(context.Background())) batches := sink.received() require.Len(t, batches, 2) @@ -85,7 +85,7 @@ func TestUsageReporter_PublishersStayPerSession(t *testing.T) { r.open("p1", usageSession{kid: "k1", publishes: true}) r.open("p2", usageSession{kid: "k1", publishes: true}) r.revalidated("p1", Bytes{Received: 50}, true) - require.NoError(t, r.flush(context.Background())) + require.NoError(t, r.meter.Flush(context.Background())) var got []string for _, rec := range sink.received()[0] { @@ -108,7 +108,7 @@ func TestUsageReporter_Publishing(t *testing.T) { r.publishing("p1") r.publishing("unknown") r.revalidated("v1", Bytes{Received: 40}, true) - require.NoError(t, r.flush(context.Background())) + require.NoError(t, r.meter.Flush(context.Background())) var got []string for _, rec := range sink.received()[0] { diff --git a/internal/auth/verifier_test.go b/internal/auth/verifier_test.go index b16478d4..605f2d2e 100644 --- a/internal/auth/verifier_test.go +++ b/internal/auth/verifier_test.go @@ -374,7 +374,7 @@ func TestVerifier_ReportsUsage(t *testing.T) { end := sessionReq(EventEnd, jwt) end.Bytes, end.Reason = Bytes{Sent: 30, Received: 40}, "closed" require.NoError(t, v.End(context.Background(), end)) - require.NoError(t, v.usage.flush(context.Background())) + require.NoError(t, v.usage.meter.Flush(context.Background())) batches := sink.received() require.Len(t, batches, 1) @@ -407,7 +407,7 @@ func TestUsageReporter(t *testing.T) { r.open("s1", testUsageSession) r.revalidated("s1", Bytes{Sent: 1, Received: 10}, true) r.revalidated("s1", Bytes{Sent: 2, Received: 20}, true) - require.NoError(t, r.flush(context.Background())) + require.NoError(t, r.meter.Flush(context.Background())) batches := sink.received() require.Len(t, batches, 1) require.Len(t, batches[0], 2) @@ -420,9 +420,9 @@ func TestUsageReporter(t *testing.T) { r := newTestUsage(t, sink) r.open("s1", testUsageSession) r.revalidated("s1", Bytes{Received: 10}, true) - assert.Error(t, r.flush(context.Background())) + assert.Error(t, r.meter.Flush(context.Background())) r.revalidated("s1", Bytes{Received: 20}, true) - require.NoError(t, r.flush(context.Background())) + require.NoError(t, r.meter.Flush(context.Background())) retry := sink.received()[1] require.Len(t, retry, 2) assert.Equal(t, usage.SessionOpen, retry[0].Type, "the open is resent") @@ -437,7 +437,7 @@ func TestUsageReporter(t *testing.T) { r := newTestUsage(t, sink) assert.False(t, r.close("nope", Bytes{Sent: 1}, "")) r.revalidated("nope", Bytes{Sent: 1}, true) - require.NoError(t, r.flush(context.Background())) + require.NoError(t, r.meter.Flush(context.Background())) assert.Empty(t, sink.received(), "nothing pending, nothing sent") }) @@ -549,7 +549,7 @@ func TestVerifier_RevalidateAfterEnd(t *testing.T) { late.Bytes = Bytes{Sent: 9} _, err = v.Authorize(context.Background(), late) require.NoError(t, err) - require.NoError(t, v.usage.flush(context.Background())) + require.NoError(t, v.usage.meter.Flush(context.Background())) var types []usage.Type for _, rec := range sink.received()[0] { @@ -822,13 +822,13 @@ func TestVerifier_Ticket_PublishingPromotesAViewer(t *testing.T) { require.NoError(t, err) _, err = v.Ticket(speak, k.ID, false) require.NoError(t, err) - require.NoError(t, v.usage.flush(context.Background())) + require.NoError(t, v.usage.meter.Flush(context.Background())) before := sink.received() v.Publishing(rv.ID) end := sessionReq(EventEnd, viewer) end.Bytes, end.Reason = Bytes{Sent: 170, Received: 15}, "closed" require.NoError(t, v.End(context.Background(), end)) - require.NoError(t, v.usage.flush(context.Background())) + require.NoError(t, v.usage.meter.Flush(context.Background())) require.Len(t, before, 1) require.Len(t, before[0], 1, "a viewer, and a ticket only verified, open no session") @@ -918,7 +918,7 @@ func TestVerifier_AnonymousUsage(t *testing.T) { req.Bytes = Bytes{Received: 40} _, err = v.Authorize(ctx, req) require.NoError(t, err) - require.NoError(t, v.usage.flush(ctx)) + require.NoError(t, v.usage.meter.Flush(ctx)) var got []string for _, rec := range sink.received()[0] { @@ -977,7 +977,7 @@ func TestUsageReporter_OpenAfterClose(t *testing.T) { r.publishing("s1") assert.Empty(t, r.sessions, "an open after the close is ignored") - require.NoError(t, r.flush(context.Background())) + require.NoError(t, r.meter.Flush(context.Background())) assert.Empty(t, sink.received(), "no open is sent") now = now.Add(sessionGrace + time.Second) diff --git a/internal/meter/meter.go b/internal/meter/meter.go index b41576b5..1bcb6dda 100644 --- a/internal/meter/meter.go +++ b/internal/meter/meter.go @@ -22,9 +22,7 @@ import ( "io" "log/slog" "maps" - "net" "net/http" - "net/url" "sync" "time" @@ -78,7 +76,6 @@ type Reporter struct { client *http.Client // refusals logs the receiver's 401s, the first of a run only. refusals reqsign.Refusals - now func() time.Time // runID names this run of the server in its totals: a restarted server // starts new totals rather than lowering old ones. runID string @@ -93,7 +90,7 @@ type Reporter struct { // New returns a Reporter sending to cfg.URL. Run sends its records. func New(cfg Config) (*Reporter, error) { - if err := checkURL(cfg.URL); err != nil { + if err := reqsign.CheckURL(cfg.URL); err != nil { return nil, err } return &Reporter{ @@ -102,7 +99,6 @@ func New(cfg Config) (*Reporter, error) { token: cfg.Token, signer: cfg.Signer, client: reqsign.NoRedirectClient(sendTimeout), - now: time.Now, runID: rand.Text(), live: map[string]bool{}, latest: map[string]usage.Record{}, @@ -110,25 +106,6 @@ func New(cfg Config) (*Reporter, error) { }, nil } -// checkURL accepts https, and http to a loopback host only. -func checkURL(rawURL string) error { - u, err := url.Parse(rawURL) - if err != nil { - return err - } - switch u.Scheme { - case "https": - case "http": - host := u.Hostname() - if ip := net.ParseIP(host); host != "localhost" && (ip == nil || !ip.IsLoopback()) { - return fmt.Errorf("http is allowed only for a loopback host, got %q", host) - } - default: - return fmt.Errorf("want an https URL, got scheme %q", u.Scheme) - } - return nil -} - // RunID returns the run named in r's totals. func (r *Reporter) RunID() string { return r.runID } @@ -198,7 +175,7 @@ func (r *Reporter) record(typ usage.Type, id, kid string) usage.Record { // timestamp is a record's ts: now, to the second. func (r *Reporter) timestamp() time.Time { - return r.now().UTC().Truncate(time.Second) + return time.Now().UTC().Truncate(time.Second) } // appendEvent queues an open or close, dropping the oldest past the cap. diff --git a/internal/meter/meter_test.go b/internal/meter/meter_test.go index 4100b766..4efe9e60 100644 --- a/internal/meter/meter_test.go +++ b/internal/meter/meter_test.go @@ -76,18 +76,12 @@ func bytesMetrics(in, out int64) map[string]int64 { return map[string]int64{usage.IngressBytes: in, usage.EgressBytes: out} } +// New takes a URL as reqsign.CheckURL does, which is tested there. func TestNew(t *testing.T) { - for rawURL, wantErr := range map[string]bool{ - "https://usage.example/v1": false, - "http://127.0.0.1:9000/v1": false, - "http://localhost/v1": false, - "http://usage.example/v1": true, - "ftp://usage.example/v1": true, - "https://usage.example/%zz": true, - } { - _, err := New(Config{URL: rawURL}) - assert.Equal(t, wantErr, err != nil, rawURL) - } + _, err := New(Config{URL: "https://usage.example/v1"}) + assert.NoError(t, err) + _, err = New(Config{URL: "http://usage.example/v1"}) + assert.Error(t, err, "a usage URL in the clear off loopback") } // A session's records go out in order: opens, usage, totals, closes, each diff --git a/internal/reqsign/reqsign.go b/internal/reqsign/reqsign.go index 6c271de0..58619728 100644 --- a/internal/reqsign/reqsign.go +++ b/internal/reqsign/reqsign.go @@ -37,7 +37,9 @@ import ( "errors" "fmt" "log/slog" + "net" "net/http" + "net/url" "strconv" "sync/atomic" "time" @@ -136,6 +138,27 @@ func signed(method, requestURI, timestamp string, body []byte) []byte { return []byte(requestContext + "\n" + method + "\n" + requestURI + "\n" + timestamp + "\n" + hex.EncodeToString(sum[:])) } +// CheckURL accepts the URL of a control-plane endpoint (a key set, a usage +// URL): https, or http only on a loopback host, so that the bearer token +// and the signature sent to it do not cross a network in the clear. +func CheckURL(rawURL string) error { + u, err := url.Parse(rawURL) + if err != nil { + return err + } + switch u.Scheme { + case "https": + case "http": + host := u.Hostname() + if ip := net.ParseIP(host); host != "localhost" && (ip == nil || !ip.IsLoopback()) { + return fmt.Errorf("http is allowed only for a loopback host, got %q", host) + } + default: + return fmt.Errorf("want an https URL, got scheme %q", u.Scheme) + } + return nil +} + // NoRedirectClient returns an HTTP client that follows no redirect: a 3xx is // answered like any other non-2xx. A client that sends signed requests must // use it: Go drops the bearer token on a redirect to another host but sends diff --git a/internal/reqsign/reqsign_test.go b/internal/reqsign/reqsign_test.go index 8d853595..84f7832a 100644 --- a/internal/reqsign/reqsign_test.go +++ b/internal/reqsign/reqsign_test.go @@ -257,3 +257,26 @@ func TestSigner_Rotate(t *testing.T) { }) } } +func TestCheckURL(t *testing.T) { + tests := map[string]struct { + url string + wantErrText string + }{ + "https": {url: "https://keys.example.com/v1/set"}, + "loopback http": {url: "http://127.0.0.1:4440/"}, + "localhost http": {url: "http://localhost:4440/"}, + "http off loopback": {url: "http://keys.example.com/", wantErrText: "loopback"}, + "another scheme": {url: "ftp://keys.example.com/", wantErrText: "https"}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + err := reqsign.CheckURL(tt.url) + + if tt.wantErrText != "" { + assert.ErrorContains(t, err, tt.wantErrText) + return + } + assert.NoError(t, err) + }) + } +}