From b332d453cb6b2075c33b9cf900ce7a904431864d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 17:51:26 +0000 Subject: [PATCH 1/7] refactor(meter): usage reporting as a package every server can use What sends usage records to QUMO_USAGE_URL moves out of the relay's verifier into internal/meter: sessions (Open, Report, Close), running totals per key (Total), coalescing, batches of 500, retries through an outage, signing and the last send at shutdown. The relay keeps only its own decisions (a publisher by itself, viewers in one total per key) in internal/auth. The records it sends are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KPWMJL7PH3rPhK5UTGavaB --- CHANGELOG.md | 4 + internal/auth/usage.go | 320 +++------------------- internal/auth/usage_viewers_test.go | 7 +- internal/auth/verifier_test.go | 61 +---- internal/meter/meter.go | 399 ++++++++++++++++++++++++++++ internal/meter/meter_test.go | 305 +++++++++++++++++++++ 6 files changed, 757 insertions(+), 339 deletions(-) create mode 100644 internal/meter/meter.go create mode 100644 internal/meter/meter_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index b9e2dcde..3e145c78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **A relay signs its key-set and usage requests, and `qumo dns` its feed reads, with its peer identity (`internal/reqsign`, #601).** With `PEER_CERT_FILE` and `PEER_KEY_FILE` (an ECDSA key; `qumo dns` takes the same two settings), each request carries `X-Relay-Certificate`, `X-Relay-Timestamp` and `X-Relay-Signature`: the key's ECDSA signature (ASN.1, base64) over SHA-256 of `qumo-relay-request`, the method, the request URI (path and query), the timestamp and the hex SHA-256 of the body, one per line, signed again on every retry. The receiver then knows which relay is calling from a key only that relay holds, and a captured request can't be changed or used for long, where a bearer token copied from a node could be reused until it is replaced. The bearer token (`QUMO_RELAY_TOKEN`, `QUMO_DNS_FEED_TOKEN`) is still sent beside it. A 401 is logged with the server's `Date` beside the local time. Funnel, HLS and ingest, which have no peer identity, are unchanged. +### Changed + +- **Usage reporting is a package of its own, for every server to use (`internal/meter`).** What sends usage records (`usage.Record`) to `QUMO_USAGE_URL` is no longer the relay's: `meter.Reporter` takes a server's sessions (`Open`, `Report` with cumulative metrics, `Close`) and running totals per key (`Total`), and keeps them in memory, coalesces each session's usage, batches by 500, retries through an outage, signs the requests when given a signer and sends what is left at shutdown, as the relay did. The relay reports through it, with what it decides itself (a publisher by itself, viewers in one total per key) kept in `internal/auth`. The records the relay sends are unchanged. + ## [v0.20.261114] - 2026-11-14 ### Added diff --git a/internal/auth/usage.go b/internal/auth/usage.go index 5261d466..0d9d461a 100644 --- a/internal/auth/usage.go +++ b/internal/auth/usage.go @@ -1,38 +1,23 @@ package auth import ( - "bytes" "context" - "crypto/rand" - "encoding/json" - "errors" - "fmt" - "io" - "log/slog" "math" - "net/http" "sync" "time" + "github.com/qumo-dev/qumo/internal/meter" "github.com/qumo-dev/qumo/internal/reqsign" "github.com/qumo-dev/qumo/usage" ) const ( - // usageFlushInterval is how often pending usage records are sent. - usageFlushInterval = 10 * time.Second - // usageSendTimeout bounds one send, and the last send at shutdown. - usageSendTimeout = 10 * time.Second - // maxBatchRecords bounds one POST, so a backlog after an outage goes out - // in pieces a receiver's body limit accepts. - maxBatchRecords = 500 - // maxPendingEvents caps the opens and closes waiting while the usage - // receiver can't be reached; the oldest are dropped past it. Usage needs - // no cap: it is one cumulative record per live session. - maxPendingEvents = 50_000 // sessionGrace is how long a session is remembered after it was last // heard of, at connect or a revalidate, for an end that arrives late. sessionGrace = 5 * time.Minute + // forgetInterval is how often sessions whose end never came are looked + // for. + forgetInterval = 10 * time.Second ) // usageSession is what the reporter remembers of a verified session. @@ -51,44 +36,21 @@ type usageSession struct { counted Bytes } -// viewerTotal is the bytes of every viewer session of one key since the relay -// started, and how much of it the receiver has acknowledged. -type viewerTotal struct { - total Bytes - reported Bytes - sent bool // reported at least once -} - -// usageReporter sends usage records (package usage) to a usage URL -// (QUMO_USAGE_URL). +// usageReporter reports the relay's sessions to a usage URL +// (QUMO_USAGE_URL), through a meter.Reporter. // // A session that may publish is reported by itself: an open when it is // admitted, its cumulative bytes at each revalidate, and a close with the // final totals. Every other session is a viewer, added up per key into one // running total for this run of the relay, sent as a viewers record. No open // or close is sent for viewers. -// -// Every record is cumulative, so a receiver can take a resent one without -// counting it twice. Records wait in memory between sends and through an -// outage; per-session usage is coalesced to its latest report. type usageReporter struct { - url string - token string - signer *reqsign.Signer // nil: requests are not signed - client *http.Client - // refusals logs the control plane's 401s, the first of a run only. - refusals reqsign.Refusals - now func() time.Time - // run names this run of the relay in its viewers records: a restarted - // relay starts new totals rather than lowering old ones. - runID string + meter *meter.Reporter + now func() time.Time mu sync.Mutex sessions map[string]usageSession - events []usage.Record // opens and closes, in order - latest map[string]usage.Record // latest cumulative report per session - viewers map[string]*viewerTotal // by kid - failing bool // the last send failed + viewers map[string]Bytes // every viewer session's bytes this run, by kid // closed is when each session that ended was closed, until it is // forgotten as a stale one would be: an open that comes after its // close, from a ticket granted as the session ended, is ignored. @@ -96,19 +58,15 @@ type usageReporter struct { } func newUsageReporter(rawURL, bearer string, signer *reqsign.Signer) (*usageReporter, error) { - if err := checkURL(rawURL); err != nil { + m, err := meter.New(meter.Config{Name: "relay", URL: rawURL, Token: bearer, Signer: signer}) + if err != nil { return nil, err } return &usageReporter{ - url: rawURL, - token: bearer, - signer: signer, - client: reqsign.NoRedirectClient(usageSendTimeout), + meter: m, now: time.Now, - runID: rand.Text(), sessions: map[string]usageSession{}, - latest: map[string]usage.Record{}, - viewers: map[string]*viewerTotal{}, + viewers: map[string]Bytes{}, closed: map[string]time.Time{}, }, nil } @@ -124,15 +82,15 @@ func (r *usageReporter) open(id string, s usageSession) { s.seen = r.now() r.sessions[id] = s if s.publishes { - r.appendEvent(r.record(usage.SessionOpen, id, s)) + r.meter.Open(id, s.kid) } } // publishing records that a viewer session may now publish, such as by a // ticket it presented: it is reported by itself from now on, opening with a // session_open. What it moved as a viewer up to its last revalidate stays in -// its key's viewer total; what it moved since is reported as its usage. A session it doesn't know, or that publishes already, is left as it -// is. +// its key's viewer total; what it moved since is reported as its usage. A +// session it doesn't know, or that publishes already, is left as it is. func (r *usageReporter) publishing(id string) { r.mu.Lock() defer r.mu.Unlock() @@ -142,7 +100,7 @@ func (r *usageReporter) publishing(id string) { } s.publishes = true r.sessions[id] = s - r.appendEvent(r.record(usage.SessionOpen, id, s)) + r.meter.Open(id, s.kid) } // revalidated records a session's cumulative bytes at a revalidate and, when @@ -162,9 +120,7 @@ func (r *usageReporter) revalidated(id string, b Bytes, live bool) { case !s.publishes: s = r.countViewer(s, b) case b.Sent > s.counted.Sent || b.Received > s.counted.Received: - rec := r.record(usage.Usage, id, s) - rec.Metrics = usageMetrics(b.since(s.counted)) - r.latest[id] = rec + r.meter.Report(id, s.kid, usageMetrics(b.since(s.counted))) } r.sessions[id] = s } @@ -184,54 +140,28 @@ func (r *usageReporter) close(id string, b Bytes, reason string) bool { r.countViewer(s, b) return true } - delete(r.latest, id) // the close carries totals at least as large - rec := r.record(usage.SessionClose, id, s) - rec.Metrics = usageMetrics(b.since(s.counted)) - rec.Reason = reason - r.appendEvent(rec) + r.meter.Close(id, s.kid, usageMetrics(b.since(s.counted)), reason) return true } -// countViewer adds what a viewer session moved since it was last -// counted to its key's viewer total, and returns the session with that -// recorded. The caller holds mu. +// countViewer adds what a viewer session moved since it was last counted to +// its key's viewer total, and returns the session with that recorded. The +// caller holds mu. func (r *usageReporter) countViewer(s usageSession, b Bytes) usageSession { t := r.viewers[s.kid] - if t == nil { - t = &viewerTotal{} - r.viewers[s.kid] = t - } if b.Sent > s.counted.Sent { - t.total.Sent += b.Sent - s.counted.Sent + t.Sent += b.Sent - s.counted.Sent s.counted.Sent = b.Sent } if b.Received > s.counted.Received { - t.total.Received += b.Received - s.counted.Received + t.Received += b.Received - s.counted.Received s.counted.Received = b.Received } + r.viewers[s.kid] = t + r.meter.Total(s.kid, usageMetrics(t)) return s } -func (r *usageReporter) record(typ usage.Type, id string, s usageSession) usage.Record { - return usage.Record{Type: typ, SessionID: id, KID: s.kid, TS: r.timestamp()} -} - -// timestamp is a record's ts: now, to the second, as the reports have always -// carried it. -func (r *usageReporter) timestamp() time.Time { - return r.now().UTC().Truncate(time.Second) -} - -// appendEvent queues an open or close, dropping the oldest past the cap. -// The caller holds mu. -func (r *usageReporter) appendEvent(rec usage.Record) { - r.events = append(r.events, rec) - if over := len(r.events) - maxPendingEvents; over > 0 { - slog.Warn("relay: usage backlog full; dropping the oldest session events", "dropped", over) - r.events = r.events[over:] - } -} - func usageMetrics(b Bytes) map[string]int64 { return map[string]int64{usage.IngressBytes: clampInt64(b.Received), usage.EgressBytes: clampInt64(b.Sent)} } @@ -243,45 +173,28 @@ func clampInt64(v uint64) int64 { return int64(v) } -// run sends pending records every usageFlushInterval until ctx ends, then -// once more. +// run sends usage until ctx ends, then once more, forgetting sessions whose +// end never came as it goes. func (r *usageReporter) run(ctx context.Context) { - ticker := time.NewTicker(usageFlushInterval) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - last, cancel := context.WithTimeout(context.WithoutCancel(ctx), usageSendTimeout) - defer cancel() - if err := r.flush(last); err != nil { - slog.Error("relay: last usage send failed; records lost", "error", err) + var wg sync.WaitGroup + wg.Go(func() { r.meter.Run(ctx) }) + wg.Go(func() { + ticker := time.NewTicker(forgetInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + r.forgetStale() } - return - case <-ticker.C: - r.forgetStale() - r.logOutcome(ctx, r.flush(ctx)) } - } + }) + wg.Wait() } -// logOutcome logs a send's first failure as a warning, repeats quietly, and -// the recovery as info: during an outage the relay retries every few -// seconds, and one line says so. -func (r *usageReporter) logOutcome(ctx context.Context, err error) { - r.mu.Lock() - wasFailing := r.failing - r.failing = err != nil - r.mu.Unlock() - switch { - case err != nil && ctx.Err() != nil: - case err != nil && !wasFailing: - slog.Warn("relay: usage send failed; keeping the records and retrying", "error", err) - case err != nil: - slog.Debug("relay: usage send still failing", "error", err) - case wasFailing: - slog.Info("relay: usage send recovered") - } -} +// flush sends every pending record now. +func (r *usageReporter) flush(ctx context.Context) error { return r.meter.Flush(ctx) } // forgetStale drops sessions whose end never came: those not heard of for // sessionGrace. @@ -292,6 +205,7 @@ func (r *usageReporter) forgetStale() { for id, s := range r.sessions { if s.seen.Before(cutoff) { delete(r.sessions, id) + r.meter.Forget(id) } } for id, at := range r.closed { @@ -301,152 +215,6 @@ func (r *usageReporter) forgetStale() { } } -// errMalformed is a 400 or 422 answer to a usage batch: the receiver can't -// read it, so resending it can't succeed. -var errMalformed = errors.New("usage batch malformed") - -// flush sends every pending record: opens, then per-session usage, then the -// viewer totals that changed, then closes, in batches of at most -// maxBatchRecords. A batch the receiver can't read (400, 422) is dropped, so -// it can't block later records. Any other failure, a 401 or 413 included, -// stops the flush and puts the unsent records back, unless newer ones -// replaced them, to try again next time. -func (r *usageReporter) flush(ctx context.Context) error { - r.mu.Lock() - events, latest := r.events, r.latest - r.events, r.latest = nil, map[string]usage.Record{} - // A viewer total is sent when it has changed since the receiver last - // acknowledged it. It stays pending until then, so it needs no requeue. - totals := map[string]Bytes{} // by kid - var viewers []usage.Record - for kid, t := range r.viewers { - if t.sent && t.total == t.reported { - continue - } - viewers = append(viewers, usage.Record{ - Type: usage.Viewers, Run: r.runID, KID: kid, Metrics: usageMetrics(t.total), TS: r.timestamp(), - }) - totals[kid] = t.total - } - r.mu.Unlock() - if len(events) == 0 && len(latest) == 0 && len(viewers) == 0 { - return nil - } - - all := make([]usage.Record, 0, len(events)+len(latest)+len(viewers)) - for _, e := range events { - if e.Type == usage.SessionOpen { - all = append(all, e) - } - } - for _, u := range latest { - all = append(all, u) - } - all = append(all, viewers...) - for _, e := range events { - if e.Type == usage.SessionClose { - all = append(all, e) - } - } - - for start := 0; start < len(all); start += maxBatchRecords { - batch := all[start:min(start+maxBatchRecords, len(all))] - err := r.send(ctx, batch) - switch { - case err == nil: - r.acknowledge(batch, totals) - case errors.Is(err, errMalformed): - slog.Error("relay: usage batch refused as malformed; dropped", "records", len(batch), "error", err) - // Not resent as it is; a viewer total goes again when it grows. - r.acknowledge(batch, totals) - default: - r.requeue(all[start:]) - return err - } - } - return nil -} - -// acknowledge records the viewer totals in a sent batch, by kid in totals, as -// reported. -func (r *usageReporter) acknowledge(batch []usage.Record, totals map[string]Bytes) { - r.mu.Lock() - defer r.mu.Unlock() - for _, rec := range batch { - if rec.Type != usage.Viewers { - continue - } - if t := r.viewers[rec.KID]; t != nil { - t.reported, t.sent = totals[rec.KID], true - } - } -} - -// requeue puts unsent records back before anything recorded since: opens -// and closes in order, and a session's usage record only if the session is -// still live and has no newer one. Viewer totals aren't put back: they stay -// pending until acknowledged. -func (r *usageReporter) requeue(unsent []usage.Record) { - r.mu.Lock() - defer r.mu.Unlock() - var events []usage.Record - for _, rec := range unsent { - switch rec.Type { - case usage.Viewers: - continue - case usage.SessionOpen, usage.SessionClose: - events = append(events, rec) - continue - } - if _, newer := r.latest[rec.SessionID]; newer { - continue - } - if _, live := r.sessions[rec.SessionID]; live { - r.latest[rec.SessionID] = rec - } - } - r.events = append(events, r.events...) - if over := len(r.events) - maxPendingEvents; over > 0 { - slog.Warn("relay: usage backlog full; dropping the oldest session events", "dropped", over) - r.events = r.events[over:] - } -} - -func (r *usageReporter) send(ctx context.Context, batch []usage.Record) error { - body, err := json.Marshal(batch) - if err != nil { - return fmt.Errorf("encode usage: %w", err) - } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.url, bytes.NewReader(body)) - if err != nil { - return err - } - if r.token != "" { - req.Header.Set("Authorization", "Bearer "+r.token) - } - req.Header.Set("Content-Type", "application/json") - // Signed at each send: a batch sent again after a failure carries a - // fresh timestamp. - if err := r.signer.Sign(req, body); err != nil { - return fmt.Errorf("usage report: %w", err) - } - resp, err := r.client.Do(req) - if err != nil { - return err - } - defer func() { _ = resp.Body.Close() }() // not actionable: drained below - _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, maxBody)) // not actionable: drained to reuse the connection - r.refusals.Observe(r.url, resp) - switch { - case resp.StatusCode >= 200 && resp.StatusCode < 300: - return nil - case resp.StatusCode == http.StatusBadRequest || resp.StatusCode == http.StatusUnprocessableEntity: - return fmt.Errorf("%w: status %d", errMalformed, resp.StatusCode) - default: - return fmt.Errorf("status %d", resp.StatusCode) - } -} - // since returns what b moved beyond base, each total at least zero. func (b Bytes) since(base Bytes) Bytes { return Bytes{Sent: b.Sent - min(b.Sent, base.Sent), Received: b.Received - min(b.Received, base.Received)} diff --git a/internal/auth/usage_viewers_test.go b/internal/auth/usage_viewers_test.go index 3f88402b..b2266114 100644 --- a/internal/auth/usage_viewers_test.go +++ b/internal/auth/usage_viewers_test.go @@ -36,7 +36,7 @@ func TestUsageReporter_ViewersAreTotalled(t *testing.T) { for _, rec := range first { byKID[rec.KID] = rec assert.Equal(t, usage.Viewers, rec.Type) - assert.Equal(t, r.runID, rec.Run) + assert.Equal(t, r.meter.RunID(), rec.Run) assert.Empty(t, rec.SessionID) assert.NoError(t, rec.Validate()) } @@ -95,11 +95,6 @@ func TestUsageReporter_PublishersStayPerSession(t *testing.T) { assert.ElementsMatch(t, []string{"session_open p1", "session_open p2", "usage p1"}, got) } -func TestUsageReporter_EachRunHasItsOwnTotals(t *testing.T) { - a, b := newTestUsage(t, &fakeUsageSink{}), newTestUsage(t, &fakeUsageSink{}) - assert.NotEqual(t, a.runID, b.runID, "a restarted relay starts new totals rather than lowering old ones") -} - // publishing turns a viewer into a publisher once: a session it doesn't know, // or one that publishes already, gets no second open. func TestUsageReporter_Publishing(t *testing.T) { diff --git a/internal/auth/verifier_test.go b/internal/auth/verifier_test.go index d02d3f32..b16478d4 100644 --- a/internal/auth/verifier_test.go +++ b/internal/auth/verifier_test.go @@ -6,7 +6,6 @@ import ( "encoding/base64" "encoding/json" "errors" - "fmt" "net/http" "net/http/httptest" "net/url" @@ -430,61 +429,8 @@ func TestUsageReporter(t *testing.T) { assert.Equal(t, int64(20), retry[1].Metrics["gateway.ingress_bytes"], "the newer report wins") }) - for _, status := range []int{http.StatusBadRequest, http.StatusUnprocessableEntity} { - t.Run(fmt.Sprintf("a malformed batch (%d) is dropped", status), func(t *testing.T) { - sink := &fakeUsageSink{statuses: []int{status, http.StatusOK}} - r := newTestUsage(t, sink) - r.open("s1", testUsageSession) - require.NoError(t, r.flush(context.Background())) - require.NoError(t, r.flush(context.Background())) - assert.Len(t, sink.received(), 1) - }) - } - - // A rotated token (401, 403), a body over the receiver's limit (413), a - // timeout (408) or throttling (429) lose nothing: the records are kept. - for _, status := range []int{http.StatusUnauthorized, http.StatusForbidden, http.StatusRequestTimeout, - http.StatusRequestEntityTooLarge, http.StatusTooManyRequests, http.StatusBadGateway} { - t.Run(fmt.Sprintf("%d keeps the records", status), func(t *testing.T) { - sink := &fakeUsageSink{statuses: []int{status, http.StatusOK}} - r := newTestUsage(t, sink) - r.open("s1", testUsageSession) - assert.Error(t, r.flush(context.Background())) - require.NoError(t, r.flush(context.Background())) - batches := sink.received() - require.Len(t, batches, 2) - assert.Equal(t, batches[0], batches[1], "the same records again") - }) - } - - t.Run("a backlog goes out in batches", func(t *testing.T) { - sink := &fakeUsageSink{} - r := newTestUsage(t, sink) - for i := range 1200 { - r.open(fmt.Sprintf("s%d", i), testUsageSession) - } - require.NoError(t, r.flush(context.Background())) - var sizes []int - for _, b := range sink.received() { - sizes = append(sizes, len(b)) - } - assert.Equal(t, []int{500, 500, 200}, sizes) - }) - - t.Run("a failure mid-backlog resends only what wasn't sent", func(t *testing.T) { - sink := &fakeUsageSink{statuses: []int{http.StatusOK, http.StatusServiceUnavailable, http.StatusOK}} - r := newTestUsage(t, sink) - for i := range 700 { - r.open(fmt.Sprintf("s%d", i), testUsageSession) - } - assert.Error(t, r.flush(context.Background())) - require.NoError(t, r.flush(context.Background())) - var sizes []int - for _, b := range sink.received() { - sizes = append(sizes, len(b)) - } - assert.Equal(t, []int{500, 200, 200}, sizes, "the first 500 went through; the next 200 are resent once") - }) + // Batching, retries and what each answer from the receiver does are + // package meter's, tested there. t.Run("an unknown session's end and usage are ignored", func(t *testing.T) { sink := &fakeUsageSink{} @@ -1031,7 +977,8 @@ func TestUsageReporter_OpenAfterClose(t *testing.T) { r.publishing("s1") assert.Empty(t, r.sessions, "an open after the close is ignored") - assert.Empty(t, r.events) + require.NoError(t, r.flush(context.Background())) + assert.Empty(t, sink.received(), "no open is sent") now = now.Add(sessionGrace + time.Second) r.forgetStale() diff --git a/internal/meter/meter.go b/internal/meter/meter.go new file mode 100644 index 00000000..b41576b5 --- /dev/null +++ b/internal/meter/meter.go @@ -0,0 +1,399 @@ +// Package meter reports usage records (package usage) to a usage URL +// (QUMO_USAGE_URL), for any qumo server that meters what its tenants use. +// +// A server opens a session for each thing it meters under a tenant's key, +// such as a relay session or an ingested stream, reports its cumulative +// metrics while it lasts, and closes it with its final totals. Usage that +// isn't worth a record per session, such as a relay's viewers, goes in one +// running total per key for the run of the server. Every record is +// cumulative, so a receiver can take a resent one without counting it twice. +// +// Records wait in memory between sends, every 10 s, and through an outage; a +// session's usage is coalesced to its latest report. +package meter + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "maps" + "net" + "net/http" + "net/url" + "sync" + "time" + + "github.com/qumo-dev/qumo/internal/reqsign" + "github.com/qumo-dev/qumo/usage" +) + +const ( + // flushInterval is how often pending records are sent. + flushInterval = 10 * time.Second + // sendTimeout bounds one send, and the last send at shutdown. + sendTimeout = 10 * time.Second + // maxBatchRecords bounds one POST, so a backlog after an outage goes out + // in pieces a receiver's body limit accepts. + maxBatchRecords = 500 + // maxPendingEvents caps the opens and closes waiting while the receiver + // can't be reached; the oldest are dropped past it. Usage needs no cap: + // it is one cumulative record per live session. + maxPendingEvents = 50_000 + // maxBody bounds how much of a reply's body is drained. + maxBody = 64 << 10 +) + +// Config is where a Reporter sends its records. +type Config struct { + // Name says who reports, in the log: "relay", "ingest". + Name string + // URL receives the records: https, or http to a loopback host. + URL string + // Token, when set, is sent as a bearer token. + Token string + // Signer, when set, signs each request with the server's identity. + Signer *reqsign.Signer +} + +// total is one key's running total for this run, and what the receiver has +// acknowledged of it. +type total struct { + metrics map[string]int64 + reported map[string]int64 + sent bool // reported at least once +} + +// A Reporter sends usage records to a usage URL. Its methods are safe for +// concurrent use. +type Reporter struct { + name string + url string + token string + signer *reqsign.Signer // nil: requests are not signed + client *http.Client + // refusals logs the receiver's 401s, the first of a run only. + refusals reqsign.Refusals + now func() time.Time + // runID names this run of the server in its totals: a restarted server + // starts new totals rather than lowering old ones. + runID string + + mu sync.Mutex + live map[string]bool // sessions opened, and not closed or forgotten + events []usage.Record // opens and closes, in order + latest map[string]usage.Record // latest cumulative report per session + totals map[string]*total // by kid + failing bool // the last send failed +} + +// New returns a Reporter sending to cfg.URL. Run sends its records. +func New(cfg Config) (*Reporter, error) { + if err := checkURL(cfg.URL); err != nil { + return nil, err + } + return &Reporter{ + name: cfg.Name, + url: cfg.URL, + token: cfg.Token, + signer: cfg.Signer, + client: reqsign.NoRedirectClient(sendTimeout), + now: time.Now, + runID: rand.Text(), + live: map[string]bool{}, + latest: map[string]usage.Record{}, + totals: map[string]*total{}, + }, nil +} + +// checkURL accepts https, and http to a loopback host only. +func checkURL(rawURL string) error { + u, err := url.Parse(rawURL) + if err != nil { + return err + } + switch u.Scheme { + case "https": + case "http": + host := u.Hostname() + if ip := net.ParseIP(host); host != "localhost" && (ip == nil || !ip.IsLoopback()) { + return fmt.Errorf("http is allowed only for a loopback host, got %q", host) + } + default: + return fmt.Errorf("want an https URL, got scheme %q", u.Scheme) + } + return nil +} + +// RunID returns the run named in r's totals. +func (r *Reporter) RunID() string { return r.runID } + +// Open records that session id began under key kid: a session_open. +func (r *Reporter) Open(id, kid string) { + r.mu.Lock() + defer r.mu.Unlock() + r.live[id] = true + r.appendEvent(r.record(usage.SessionOpen, id, kid)) +} + +// Report records the cumulative metrics of session id so far: a usage +// record, replacing any not yet sent. A session not open is ignored. +func (r *Reporter) Report(id, kid string, metrics map[string]int64) { + r.mu.Lock() + defer r.mu.Unlock() + if !r.live[id] { + return + } + rec := r.record(usage.Usage, id, kid) + rec.Metrics = maps.Clone(metrics) + r.latest[id] = rec +} + +// Close records the end of session id with its final totals and the reason +// it ended: a session_close, which supersedes its pending usage. A session +// not open is ignored. +func (r *Reporter) Close(id, kid string, metrics map[string]int64, reason string) { + r.mu.Lock() + defer r.mu.Unlock() + if !r.live[id] { + return + } + delete(r.live, id) + delete(r.latest, id) // the close carries totals at least as large + rec := r.record(usage.SessionClose, id, kid) + rec.Metrics = maps.Clone(metrics) + rec.Reason = reason + r.appendEvent(rec) +} + +// Forget drops session id without a close, for one whose end never came. Its +// usage not yet sent still goes, but is not sent again after a failure. +func (r *Reporter) Forget(id string) { + r.mu.Lock() + defer r.mu.Unlock() + delete(r.live, id) +} + +// Total records the running total of key kid for this run: a total record +// (usage.Viewers), sent whenever it differs from what the receiver last +// acknowledged. +func (r *Reporter) Total(kid string, metrics map[string]int64) { + r.mu.Lock() + defer r.mu.Unlock() + t := r.totals[kid] + if t == nil { + t = &total{} + r.totals[kid] = t + } + t.metrics = maps.Clone(metrics) +} + +func (r *Reporter) record(typ usage.Type, id, kid string) usage.Record { + return usage.Record{Type: typ, SessionID: id, KID: kid, TS: r.timestamp()} +} + +// timestamp is a record's ts: now, to the second. +func (r *Reporter) timestamp() time.Time { + return r.now().UTC().Truncate(time.Second) +} + +// appendEvent queues an open or close, dropping the oldest past the cap. +// The caller holds mu. +func (r *Reporter) appendEvent(rec usage.Record) { + r.events = append(r.events, rec) + r.capEvents() +} + +// capEvents drops the oldest events past maxPendingEvents. The caller holds +// mu. +func (r *Reporter) capEvents() { + if over := len(r.events) - maxPendingEvents; over > 0 { + slog.Warn(r.name+": usage backlog full; dropping the oldest session events", "dropped", over) + r.events = r.events[over:] + } +} + +// Run sends pending records every 10 s until ctx ends, then once more. +func (r *Reporter) Run(ctx context.Context) { + ticker := time.NewTicker(flushInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + last, cancel := context.WithTimeout(context.WithoutCancel(ctx), sendTimeout) + defer cancel() + if err := r.Flush(last); err != nil { + slog.Error(r.name+": last usage send failed; records lost", "error", err) + } + return + case <-ticker.C: + r.logOutcome(ctx, r.Flush(ctx)) + } + } +} + +// logOutcome logs a send's first failure as a warning, repeats quietly, and +// the recovery as info: during an outage the server retries every few +// seconds, and one line says so. +func (r *Reporter) logOutcome(ctx context.Context, err error) { + r.mu.Lock() + wasFailing := r.failing + r.failing = err != nil + r.mu.Unlock() + switch { + case err != nil && ctx.Err() != nil: + case err != nil && !wasFailing: + slog.Warn(r.name+": usage send failed; keeping the records and retrying", "error", err) + case err != nil: + slog.Debug(r.name+": usage send still failing", "error", err) + case wasFailing: + slog.Info(r.name + ": usage send recovered") + } +} + +// errMalformed is a 400 or 422 answer to a batch: the receiver can't read +// it, so resending it can't succeed. +var errMalformed = errors.New("usage batch malformed") + +// Flush sends every pending record: opens, then per-session usage, then the +// totals that changed, then closes, in batches of at most 500. A batch the +// receiver can't read (400, 422) is dropped, so it can't block later +// records. Any other failure, a 401 or 413 included, stops the flush and puts +// the unsent records back, unless newer ones replaced them, to try again +// next time. +func (r *Reporter) Flush(ctx context.Context) error { + r.mu.Lock() + events, latest := r.events, r.latest + r.events, r.latest = nil, map[string]usage.Record{} + // A total is sent when it has changed since the receiver last + // acknowledged it. It stays pending until then, so it needs no requeue. + sending := map[string]map[string]int64{} // by kid + var totals []usage.Record + for kid, t := range r.totals { + if t.sent && maps.Equal(t.metrics, t.reported) { + continue + } + totals = append(totals, usage.Record{ + Type: usage.Viewers, Run: r.runID, KID: kid, Metrics: maps.Clone(t.metrics), TS: r.timestamp(), + }) + sending[kid] = t.metrics + } + r.mu.Unlock() + if len(events) == 0 && len(latest) == 0 && len(totals) == 0 { + return nil + } + + all := make([]usage.Record, 0, len(events)+len(latest)+len(totals)) + for _, e := range events { + if e.Type == usage.SessionOpen { + all = append(all, e) + } + } + for _, u := range latest { + all = append(all, u) + } + all = append(all, totals...) + for _, e := range events { + if e.Type == usage.SessionClose { + all = append(all, e) + } + } + + for start := 0; start < len(all); start += maxBatchRecords { + batch := all[start:min(start+maxBatchRecords, len(all))] + err := r.send(ctx, batch) + switch { + case err == nil: + r.acknowledge(batch, sending) + case errors.Is(err, errMalformed): + slog.Error(r.name+": usage batch refused as malformed; dropped", "records", len(batch), "error", err) + // Not resent as it is; a total goes again when it changes. + r.acknowledge(batch, sending) + default: + r.requeue(all[start:]) + return err + } + } + return nil +} + +// acknowledge records the totals in a sent batch, by kid in sending, as +// reported. +func (r *Reporter) acknowledge(batch []usage.Record, sending map[string]map[string]int64) { + r.mu.Lock() + defer r.mu.Unlock() + for _, rec := range batch { + if rec.Type != usage.Viewers { + continue + } + if t := r.totals[rec.KID]; t != nil { + t.reported, t.sent = sending[rec.KID], true + } + } +} + +// requeue puts unsent records back before anything recorded since: opens +// and closes in order, and a session's usage record only if the session is +// still live and has no newer one. Totals aren't put back: they stay pending +// until acknowledged. +func (r *Reporter) requeue(unsent []usage.Record) { + r.mu.Lock() + defer r.mu.Unlock() + var events []usage.Record + for _, rec := range unsent { + switch rec.Type { + case usage.Viewers: + continue + case usage.SessionOpen, usage.SessionClose: + events = append(events, rec) + continue + } + if _, newer := r.latest[rec.SessionID]; newer { + continue + } + if r.live[rec.SessionID] { + r.latest[rec.SessionID] = rec + } + } + r.events = append(events, r.events...) + r.capEvents() +} + +func (r *Reporter) send(ctx context.Context, batch []usage.Record) error { + body, err := json.Marshal(batch) + if err != nil { + return fmt.Errorf("encode usage: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.url, bytes.NewReader(body)) + if err != nil { + return err + } + if r.token != "" { + req.Header.Set("Authorization", "Bearer "+r.token) + } + req.Header.Set("Content-Type", "application/json") + // Signed at each send: a batch sent again after a failure carries a + // fresh timestamp. + if err := r.signer.Sign(req, body); err != nil { + return fmt.Errorf("usage report: %w", err) + } + resp, err := r.client.Do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() // not actionable: drained below + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, maxBody)) // not actionable: drained to reuse the connection + r.refusals.Observe(r.url, resp) + switch { + case resp.StatusCode >= 200 && resp.StatusCode < 300: + return nil + case resp.StatusCode == http.StatusBadRequest || resp.StatusCode == http.StatusUnprocessableEntity: + return fmt.Errorf("%w: status %d", errMalformed, resp.StatusCode) + default: + return fmt.Errorf("status %d", resp.StatusCode) + } +} diff --git a/internal/meter/meter_test.go b/internal/meter/meter_test.go new file mode 100644 index 00000000..4100b766 --- /dev/null +++ b/internal/meter/meter_test.go @@ -0,0 +1,305 @@ +package meter + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "sync" + "testing" + "testing/synctest" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/qumo-dev/qumo/usage" +) + +// fakeSink records the batches POSTed to it and answers from a status queue: +// in order, the last repeating; empty means 200. +type fakeSink struct { + mu sync.Mutex + statuses []int + batches [][]usage.Record + gotAuth string +} + +func (f *fakeSink) ServeHTTP(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + f.gotAuth = r.Header.Get("Authorization") + var batch []usage.Record + if err := json.NewDecoder(r.Body).Decode(&batch); err == nil { + f.batches = append(f.batches, batch) + } + status := http.StatusOK + if len(f.statuses) > 0 { + status = f.statuses[0] + if len(f.statuses) > 1 { + f.statuses = f.statuses[1:] + } + } + w.WriteHeader(status) +} + +func (f *fakeSink) received() [][]usage.Record { + f.mu.Lock() + defer f.mu.Unlock() + return append([][]usage.Record(nil), f.batches...) +} + +func newTestReporter(t *testing.T, sink *fakeSink) *Reporter { + t.Helper() + srv := httptest.NewServer(sink) + t.Cleanup(srv.Close) + r, err := New(Config{Name: "test", URL: srv.URL, Token: "t"}) + require.NoError(t, err) + return r +} + +// summary is each record of batch as "type session_id" or "type kid". +func summary(batch []usage.Record) []string { + var out []string + for _, rec := range batch { + id := rec.SessionID + if rec.Type == usage.Viewers { + id = rec.KID + } + out = append(out, string(rec.Type)+" "+id) + } + return out +} + +func bytesMetrics(in, out int64) map[string]int64 { + return map[string]int64{usage.IngressBytes: in, usage.EgressBytes: out} +} + +func TestNew(t *testing.T) { + for rawURL, wantErr := range map[string]bool{ + "https://usage.example/v1": false, + "http://127.0.0.1:9000/v1": false, + "http://localhost/v1": false, + "http://usage.example/v1": true, + "ftp://usage.example/v1": true, + "https://usage.example/%zz": true, + } { + _, err := New(Config{URL: rawURL}) + assert.Equal(t, wantErr, err != nil, rawURL) + } +} + +// A session's records go out in order: opens, usage, totals, closes, each +// valid, with the bearer token. +func TestReporter_Records(t *testing.T) { + sink := &fakeSink{} + r := newTestReporter(t, sink) + r.Open("s1", "k1") + r.Open("s2", "k1") + r.Report("s1", "k1", bytesMetrics(10, 1)) + r.Report("s1", "k1", bytesMetrics(20, 2)) + r.Total("k2", bytesMetrics(0, 7)) + r.Close("s2", "k1", bytesMetrics(5, 5), usage.ReasonClosed) + + require.NoError(t, r.Flush(context.Background())) + + batches := sink.received() + require.Len(t, batches, 1) + assert.Equal(t, []string{"session_open s1", "session_open s2", "usage s1", "viewers k2", "session_close s2"}, summary(batches[0])) + for _, rec := range batches[0] { + assert.NoError(t, rec.Validate()) + } + assert.Equal(t, bytesMetrics(20, 2), batches[0][2].Metrics, "usage is coalesced to the latest report") + assert.Equal(t, r.RunID(), batches[0][3].Run) + assert.Equal(t, usage.ReasonClosed, batches[0][4].Reason) + assert.Equal(t, "Bearer t", sink.gotAuth) +} + +func TestReporter_CloseSupersedesUsage(t *testing.T) { + sink := &fakeSink{} + r := newTestReporter(t, sink) + r.Open("s1", "k1") + r.Report("s1", "k1", bytesMetrics(10, 1)) + r.Close("s1", "k1", bytesMetrics(30, 3), usage.ReasonClosed) + + require.NoError(t, r.Flush(context.Background())) + + assert.Equal(t, []string{"session_open s1", "session_close s1"}, summary(sink.received()[0])) +} + +// Usage and a close for a session not open, closed already or forgotten are +// ignored. +func TestReporter_NotOpen(t *testing.T) { + sink := &fakeSink{} + r := newTestReporter(t, sink) + r.Report("nope", "k1", bytesMetrics(1, 1)) + r.Close("nope", "k1", bytesMetrics(1, 1), usage.ReasonClosed) + r.Open("s1", "k1") + r.Close("s1", "k1", bytesMetrics(1, 1), usage.ReasonClosed) + r.Close("s1", "k1", bytesMetrics(2, 2), usage.ReasonClosed) + r.Report("s1", "k1", bytesMetrics(3, 3)) + r.Open("s2", "k1") + r.Forget("s2") + r.Report("s2", "k1", bytesMetrics(3, 3)) + + require.NoError(t, r.Flush(context.Background())) + + assert.Equal(t, []string{"session_open s1", "session_open s2", "session_close s1"}, summary(sink.received()[0])) +} + +func TestReporter_FailedSendIsRetriedWithNewerUsage(t *testing.T) { + sink := &fakeSink{statuses: []int{http.StatusBadGateway, http.StatusOK}} + r := newTestReporter(t, sink) + r.Open("s1", "k1") + r.Report("s1", "k1", bytesMetrics(10, 0)) + assert.Error(t, r.Flush(context.Background())) + r.Report("s1", "k1", bytesMetrics(20, 0)) + + require.NoError(t, r.Flush(context.Background())) + + retry := sink.received()[1] + assert.Equal(t, []string{"session_open s1", "usage s1"}, summary(retry), "the open is resent") + assert.Equal(t, int64(20), retry[1].Metrics[usage.IngressBytes], "the newer report wins") +} + +// A forgotten session's usage goes once, but not again after a failure. +func TestReporter_ForgottenUsageIsNotResent(t *testing.T) { + sink := &fakeSink{statuses: []int{http.StatusOK, http.StatusBadGateway, http.StatusOK}} + r := newTestReporter(t, sink) + r.Open("s1", "k1") + require.NoError(t, r.Flush(context.Background())) + r.Report("s1", "k1", bytesMetrics(10, 0)) + r.Forget("s1") + + assert.Error(t, r.Flush(context.Background())) + require.NoError(t, r.Flush(context.Background())) + + assert.Len(t, sink.received(), 2, "the open, then the usage once; nothing to resend") + assert.Equal(t, []string{"usage s1"}, summary(sink.received()[1])) +} + +func TestReporter_MalformedBatchIsDropped(t *testing.T) { + for _, status := range []int{http.StatusBadRequest, http.StatusUnprocessableEntity} { + t.Run(fmt.Sprint(status), func(t *testing.T) { + sink := &fakeSink{statuses: []int{status, http.StatusOK}} + r := newTestReporter(t, sink) + r.Open("s1", "k1") + require.NoError(t, r.Flush(context.Background())) + require.NoError(t, r.Flush(context.Background())) + assert.Len(t, sink.received(), 1) + }) + } +} + +// A rotated token (401, 403), a body over the receiver's limit (413), a +// timeout (408) or throttling (429) lose nothing: the records are kept. +func TestReporter_FailureKeepsTheRecords(t *testing.T) { + for _, status := range []int{http.StatusUnauthorized, http.StatusForbidden, http.StatusRequestTimeout, + http.StatusRequestEntityTooLarge, http.StatusTooManyRequests, http.StatusBadGateway} { + t.Run(fmt.Sprint(status), func(t *testing.T) { + sink := &fakeSink{statuses: []int{status, http.StatusOK}} + r := newTestReporter(t, sink) + r.Open("s1", "k1") + assert.Error(t, r.Flush(context.Background())) + require.NoError(t, r.Flush(context.Background())) + batches := sink.received() + require.Len(t, batches, 2) + assert.Equal(t, batches[0], batches[1], "the same records again") + }) + } +} + +func TestReporter_Batches(t *testing.T) { + t.Run("a backlog goes out in batches", func(t *testing.T) { + sink := &fakeSink{} + r := newTestReporter(t, sink) + for i := range 1200 { + r.Open(fmt.Sprintf("s%d", i), "k1") + } + require.NoError(t, r.Flush(context.Background())) + var sizes []int + for _, b := range sink.received() { + sizes = append(sizes, len(b)) + } + assert.Equal(t, []int{500, 500, 200}, sizes) + }) + + t.Run("a failure mid-backlog resends only what wasn't sent", func(t *testing.T) { + sink := &fakeSink{statuses: []int{http.StatusOK, http.StatusServiceUnavailable, http.StatusOK}} + r := newTestReporter(t, sink) + for i := range 700 { + r.Open(fmt.Sprintf("s%d", i), "k1") + } + assert.Error(t, r.Flush(context.Background())) + require.NoError(t, r.Flush(context.Background())) + var sizes []int + for _, b := range sink.received() { + sizes = append(sizes, len(b)) + } + assert.Equal(t, []int{500, 200, 200}, sizes, "the first 500 went through; the next 200 are resent once") + }) +} + +func TestReporter_Totals(t *testing.T) { + sink := &fakeSink{statuses: []int{http.StatusOK, http.StatusServiceUnavailable, http.StatusOK}} + r := newTestReporter(t, sink) + r.Total("k1", bytesMetrics(0, 100)) + require.NoError(t, r.Flush(context.Background())) + + r.Total("k1", bytesMetrics(0, 100)) + require.NoError(t, r.Flush(context.Background())) + assert.Len(t, sink.received(), 1, "an unchanged total isn't sent again") + + r.Total("k1", bytesMetrics(0, 150)) + assert.Error(t, r.Flush(context.Background())) + r.Total("k1", bytesMetrics(0, 160)) + require.NoError(t, r.Flush(context.Background())) + + batches := sink.received() + require.Len(t, batches, 3) + require.Len(t, batches[2], 1, "one record, not the old total and the new") + assert.Equal(t, int64(160), batches[2][0].Metrics[usage.EgressBytes]) +} + +func TestReporter_EachRunHasItsOwnTotals(t *testing.T) { + a, b := newTestReporter(t, &fakeSink{}), newTestReporter(t, &fakeSink{}) + assert.NotEqual(t, a.RunID(), b.RunID(), "a restarted server starts new totals rather than lowering old ones") +} + +// Run sends every flushInterval, and once more when ctx ends. +func TestReporter_Run(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var mu sync.Mutex + var sent []time.Time + r, err := New(Config{Name: "test", URL: "http://127.0.0.1/usage"}) + require.NoError(t, err) + r.client = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + mu.Lock() + defer mu.Unlock() + sent = append(sent, time.Now()) + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody}, nil + })} + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan struct{}) + start := time.Now() + go func() { + defer close(done) + r.Run(ctx) + }() + + r.Open("s1", "k1") + time.Sleep(flushInterval + time.Second) + r.Close("s1", "k1", bytesMetrics(1, 1), usage.ReasonClosed) + time.Sleep(time.Second) + cancel() + <-done + + assert.Equal(t, []time.Time{start.Add(flushInterval), start.Add(flushInterval + 2*time.Second)}, sent, + "at the tick, then the close at shutdown") + }) +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) } From 99c79429532fe996b2a097891cd536483feda6a6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 19:26:21 +0000 Subject: [PATCH 2/7] docs(changelog): keep the meter entry under Unreleased, not v0.21.261118 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KPWMJL7PH3rPhK5UTGavaB --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2160476..31c0c5c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- **Usage reporting is a package of its own, for every server to use (`internal/meter`).** What sends usage records (`usage.Record`) to `QUMO_USAGE_URL` is no longer the relay's: `meter.Reporter` takes a server's sessions (`Open`, `Report` with cumulative metrics, `Close`) and running totals per key (`Total`), and keeps them in memory, coalesces each session's usage, batches by 500, retries through an outage, signs the requests when given a signer and sends what is left at shutdown, as the relay did. The relay reports through it, with what it decides itself (a publisher by itself, viewers in one total per key) kept in `internal/auth`. The records the relay sends are unchanged. + ## [v0.21.261118] - 2026-11-18 ### Security @@ -17,10 +21,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **A relay signs its key-set and usage requests, and `qumo dns` its feed reads, with its peer identity (`internal/reqsign`, #601).** With `PEER_CERT_FILE` and `PEER_KEY_FILE` (an ECDSA key; `qumo dns` takes the same two settings), each request carries `X-Relay-Certificate`, `X-Relay-Timestamp` and `X-Relay-Signature`: the key's ECDSA signature (ASN.1, base64) over SHA-256 of `qumo-relay-request`, the method, the request URI (path and query), the timestamp and the hex SHA-256 of the body, one per line, signed again on every retry. The receiver then knows which relay is calling from a key only that relay holds, and a captured request can't be changed or used for long, where a bearer token copied from a node could be reused until it is replaced. The bearer token (`QUMO_RELAY_TOKEN`, `QUMO_DNS_FEED_TOKEN`) is still sent beside it. A 401 is logged with the server's `Date` beside the local time. Funnel, HLS and ingest, which have no peer identity, are unchanged. -### Changed - -- **Usage reporting is a package of its own, for every server to use (`internal/meter`).** What sends usage records (`usage.Record`) to `QUMO_USAGE_URL` is no longer the relay's: `meter.Reporter` takes a server's sessions (`Open`, `Report` with cumulative metrics, `Close`) and running totals per key (`Total`), and keeps them in memory, coalesces each session's usage, batches by 500, retries through an outage, signs the requests when given a signer and sends what is left at shutdown, as the relay did. The relay reports through it, with what it decides itself (a publisher by itself, viewers in one total per key) kept in `internal/auth`. The records the relay sends are unchanged. - ## [v0.20.261114] - 2026-11-14 ### Added From 2dda76d854775f1ef6ec061f0238fe2e3f4756e9 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 11:03:21 +0000 Subject: [PATCH 3/7] refactor(meter): one control-plane URL check; drop hooks nothing uses - The check that a key-set or usage URL is https, or http only to a loopback host, was in internal/auth and copied into internal/meter. It is now reqsign.CheckURL, beside the other rules for requests to the control plane (NoRedirectClient, Refusals), so the two can't drift. - meter.Reporter.now was never overridden; it uses time.Now. - usageReporter.flush existed only for tests; they call meter.Flush. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KPWMJL7PH3rPhK5UTGavaB --- internal/auth/auth.go | 22 ---------------------- internal/auth/auth_test.go | 24 ------------------------ internal/auth/keysource.go | 2 +- internal/auth/signed_requests_test.go | 4 ++-- internal/auth/usage.go | 3 --- internal/auth/usage_viewers_test.go | 14 +++++++------- internal/auth/verifier_test.go | 20 ++++++++++---------- internal/meter/meter.go | 27 ++------------------------- internal/meter/meter_test.go | 16 +++++----------- internal/reqsign/reqsign.go | 23 +++++++++++++++++++++++ internal/reqsign/reqsign_test.go | 23 +++++++++++++++++++++++ 11 files changed, 73 insertions(+), 105 deletions(-) diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 7132fb05..010991cb 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -12,9 +12,7 @@ package auth import ( "errors" "fmt" - "net" "net/http" - "net/url" ) // maxBody bounds how much of a reply's body is drained. @@ -60,26 +58,6 @@ func RefusalStatus(err error) int { return http.StatusServiceUnavailable } -// checkURL accepts an https URL, or http only on a loopback host: the bearer -// token the relay sends to it must not cross a network in the clear. -func checkURL(rawURL string) error { - u, err := url.Parse(rawURL) - if err != nil { - return err - } - switch u.Scheme { - case "https": - case "http": - host := u.Hostname() - if ip := net.ParseIP(host); host != "localhost" && (ip == nil || !ip.IsLoopback()) { - return fmt.Errorf("http is allowed only for a loopback host, got %q", host) - } - default: - return fmt.Errorf("want an https URL, got scheme %q", u.Scheme) - } - return nil -} - // Request is one session event: what the relay asks its Authorize about at // connect and revalidate, and reports to its End. type Request struct { diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go index b3c80145..08bf8e0c 100644 --- a/internal/auth/auth_test.go +++ b/internal/auth/auth_test.go @@ -9,30 +9,6 @@ import ( "github.com/stretchr/testify/assert" ) -func TestCheckURL(t *testing.T) { - tests := map[string]struct { - url string - wantErrText string - }{ - "https": {url: "https://keys.example.com/v1/set"}, - "loopback http": {url: "http://127.0.0.1:4440/"}, - "localhost http": {url: "http://localhost:4440/"}, - "http off loopback": {url: "http://keys.example.com/", wantErrText: "loopback"}, - "another scheme": {url: "ftp://keys.example.com/", wantErrText: "https"}, - } - for name, tt := range tests { - t.Run(name, func(t *testing.T) { - err := checkURL(tt.url) - - if tt.wantErrText != "" { - assert.ErrorContains(t, err, tt.wantErrText) - return - } - assert.NoError(t, err) - }) - } -} - func TestRefusalStatus(t *testing.T) { tests := map[string]struct { err error diff --git a/internal/auth/keysource.go b/internal/auth/keysource.go index 421de59b..a36e9a08 100644 --- a/internal/auth/keysource.go +++ b/internal/auth/keysource.go @@ -306,7 +306,7 @@ func (u *urlKeySource) touchCache(now time.Time) { } func newURLKeySource(rawURL, bearer string, signer *reqsign.Signer) (*urlKeySource, error) { - if err := checkURL(rawURL); err != nil { + if err := reqsign.CheckURL(rawURL); err != nil { return nil, err } return &urlKeySource{url: rawURL, token: bearer, signer: signer, client: reqsign.NoRedirectClient(keysFetchTimeout)}, nil diff --git a/internal/auth/signed_requests_test.go b/internal/auth/signed_requests_test.go index 0fd0b33c..276e32a7 100644 --- a/internal/auth/signed_requests_test.go +++ b/internal/auth/signed_requests_test.go @@ -86,8 +86,8 @@ func TestVerifier_SignsItsRequests(t *testing.T) { require.NoError(t, v.source.refresh(t.Context(), &store, time.Now())) v.usage.open("s1", testUsageSession) - assert.Error(t, v.usage.flush(t.Context())) - require.NoError(t, v.usage.flush(t.Context())) + assert.Error(t, v.usage.meter.Flush(t.Context())) + require.NoError(t, v.usage.meter.Flush(t.Context())) require.Len(t, keyReqs.all(), 1) assertSigned(t, keyReqs.all()[0], "relay-token") diff --git a/internal/auth/usage.go b/internal/auth/usage.go index 0d9d461a..3832ff0d 100644 --- a/internal/auth/usage.go +++ b/internal/auth/usage.go @@ -193,9 +193,6 @@ func (r *usageReporter) run(ctx context.Context) { wg.Wait() } -// flush sends every pending record now. -func (r *usageReporter) flush(ctx context.Context) error { return r.meter.Flush(ctx) } - // forgetStale drops sessions whose end never came: those not heard of for // sessionGrace. func (r *usageReporter) forgetStale() { diff --git a/internal/auth/usage_viewers_test.go b/internal/auth/usage_viewers_test.go index b2266114..79ee16e7 100644 --- a/internal/auth/usage_viewers_test.go +++ b/internal/auth/usage_viewers_test.go @@ -28,7 +28,7 @@ func TestUsageReporter_ViewersAreTotalled(t *testing.T) { r.revalidated("v1", Bytes{Sent: 100, Received: 10}, true) r.revalidated("v2", Bytes{Sent: 200, Received: 20}, true) r.revalidated("other", Bytes{Sent: 7}, true) - require.NoError(t, r.flush(context.Background())) + require.NoError(t, r.meter.Flush(context.Background())) first := sink.received()[0] require.Len(t, first, 2, "one record per key, not per viewer") @@ -44,7 +44,7 @@ func TestUsageReporter_ViewersAreTotalled(t *testing.T) { assert.Equal(t, int64(7), byKID["k2"].Metrics["gateway.egress_bytes"]) t.Run("an unchanged total isn't sent again", func(t *testing.T) { - require.NoError(t, r.flush(context.Background())) + require.NoError(t, r.meter.Flush(context.Background())) assert.Len(t, sink.received(), 1) }) @@ -54,7 +54,7 @@ func TestUsageReporter_ViewersAreTotalled(t *testing.T) { assert.True(t, r.close("v2", Bytes{Sent: 260, Received: 25}, "closed")) // +60, +5 r.revalidated("v2", Bytes{Sent: 999}, true) // after its close: ignored assert.True(t, r.close("v3", Bytes{}, "closed")) // moved nothing - require.NoError(t, r.flush(context.Background())) + require.NoError(t, r.meter.Flush(context.Background())) batches := sink.received() require.Len(t, batches, 2) @@ -69,9 +69,9 @@ func TestUsageReporter_ViewerTotalSurvivesAFailedSend(t *testing.T) { r.open("v1", viewerSession("k1")) r.revalidated("v1", Bytes{Sent: 100}, true) - assert.Error(t, r.flush(context.Background())) + assert.Error(t, r.meter.Flush(context.Background())) r.revalidated("v1", Bytes{Sent: 160}, true) - require.NoError(t, r.flush(context.Background())) + require.NoError(t, r.meter.Flush(context.Background())) batches := sink.received() require.Len(t, batches, 2) @@ -85,7 +85,7 @@ func TestUsageReporter_PublishersStayPerSession(t *testing.T) { r.open("p1", usageSession{kid: "k1", publishes: true}) r.open("p2", usageSession{kid: "k1", publishes: true}) r.revalidated("p1", Bytes{Received: 50}, true) - require.NoError(t, r.flush(context.Background())) + require.NoError(t, r.meter.Flush(context.Background())) var got []string for _, rec := range sink.received()[0] { @@ -108,7 +108,7 @@ func TestUsageReporter_Publishing(t *testing.T) { r.publishing("p1") r.publishing("unknown") r.revalidated("v1", Bytes{Received: 40}, true) - require.NoError(t, r.flush(context.Background())) + require.NoError(t, r.meter.Flush(context.Background())) var got []string for _, rec := range sink.received()[0] { diff --git a/internal/auth/verifier_test.go b/internal/auth/verifier_test.go index b16478d4..605f2d2e 100644 --- a/internal/auth/verifier_test.go +++ b/internal/auth/verifier_test.go @@ -374,7 +374,7 @@ func TestVerifier_ReportsUsage(t *testing.T) { end := sessionReq(EventEnd, jwt) end.Bytes, end.Reason = Bytes{Sent: 30, Received: 40}, "closed" require.NoError(t, v.End(context.Background(), end)) - require.NoError(t, v.usage.flush(context.Background())) + require.NoError(t, v.usage.meter.Flush(context.Background())) batches := sink.received() require.Len(t, batches, 1) @@ -407,7 +407,7 @@ func TestUsageReporter(t *testing.T) { r.open("s1", testUsageSession) r.revalidated("s1", Bytes{Sent: 1, Received: 10}, true) r.revalidated("s1", Bytes{Sent: 2, Received: 20}, true) - require.NoError(t, r.flush(context.Background())) + require.NoError(t, r.meter.Flush(context.Background())) batches := sink.received() require.Len(t, batches, 1) require.Len(t, batches[0], 2) @@ -420,9 +420,9 @@ func TestUsageReporter(t *testing.T) { r := newTestUsage(t, sink) r.open("s1", testUsageSession) r.revalidated("s1", Bytes{Received: 10}, true) - assert.Error(t, r.flush(context.Background())) + assert.Error(t, r.meter.Flush(context.Background())) r.revalidated("s1", Bytes{Received: 20}, true) - require.NoError(t, r.flush(context.Background())) + require.NoError(t, r.meter.Flush(context.Background())) retry := sink.received()[1] require.Len(t, retry, 2) assert.Equal(t, usage.SessionOpen, retry[0].Type, "the open is resent") @@ -437,7 +437,7 @@ func TestUsageReporter(t *testing.T) { r := newTestUsage(t, sink) assert.False(t, r.close("nope", Bytes{Sent: 1}, "")) r.revalidated("nope", Bytes{Sent: 1}, true) - require.NoError(t, r.flush(context.Background())) + require.NoError(t, r.meter.Flush(context.Background())) assert.Empty(t, sink.received(), "nothing pending, nothing sent") }) @@ -549,7 +549,7 @@ func TestVerifier_RevalidateAfterEnd(t *testing.T) { late.Bytes = Bytes{Sent: 9} _, err = v.Authorize(context.Background(), late) require.NoError(t, err) - require.NoError(t, v.usage.flush(context.Background())) + require.NoError(t, v.usage.meter.Flush(context.Background())) var types []usage.Type for _, rec := range sink.received()[0] { @@ -822,13 +822,13 @@ func TestVerifier_Ticket_PublishingPromotesAViewer(t *testing.T) { require.NoError(t, err) _, err = v.Ticket(speak, k.ID, false) require.NoError(t, err) - require.NoError(t, v.usage.flush(context.Background())) + require.NoError(t, v.usage.meter.Flush(context.Background())) before := sink.received() v.Publishing(rv.ID) end := sessionReq(EventEnd, viewer) end.Bytes, end.Reason = Bytes{Sent: 170, Received: 15}, "closed" require.NoError(t, v.End(context.Background(), end)) - require.NoError(t, v.usage.flush(context.Background())) + require.NoError(t, v.usage.meter.Flush(context.Background())) require.Len(t, before, 1) require.Len(t, before[0], 1, "a viewer, and a ticket only verified, open no session") @@ -918,7 +918,7 @@ func TestVerifier_AnonymousUsage(t *testing.T) { req.Bytes = Bytes{Received: 40} _, err = v.Authorize(ctx, req) require.NoError(t, err) - require.NoError(t, v.usage.flush(ctx)) + require.NoError(t, v.usage.meter.Flush(ctx)) var got []string for _, rec := range sink.received()[0] { @@ -977,7 +977,7 @@ func TestUsageReporter_OpenAfterClose(t *testing.T) { r.publishing("s1") assert.Empty(t, r.sessions, "an open after the close is ignored") - require.NoError(t, r.flush(context.Background())) + require.NoError(t, r.meter.Flush(context.Background())) assert.Empty(t, sink.received(), "no open is sent") now = now.Add(sessionGrace + time.Second) diff --git a/internal/meter/meter.go b/internal/meter/meter.go index b41576b5..1bcb6dda 100644 --- a/internal/meter/meter.go +++ b/internal/meter/meter.go @@ -22,9 +22,7 @@ import ( "io" "log/slog" "maps" - "net" "net/http" - "net/url" "sync" "time" @@ -78,7 +76,6 @@ type Reporter struct { client *http.Client // refusals logs the receiver's 401s, the first of a run only. refusals reqsign.Refusals - now func() time.Time // runID names this run of the server in its totals: a restarted server // starts new totals rather than lowering old ones. runID string @@ -93,7 +90,7 @@ type Reporter struct { // New returns a Reporter sending to cfg.URL. Run sends its records. func New(cfg Config) (*Reporter, error) { - if err := checkURL(cfg.URL); err != nil { + if err := reqsign.CheckURL(cfg.URL); err != nil { return nil, err } return &Reporter{ @@ -102,7 +99,6 @@ func New(cfg Config) (*Reporter, error) { token: cfg.Token, signer: cfg.Signer, client: reqsign.NoRedirectClient(sendTimeout), - now: time.Now, runID: rand.Text(), live: map[string]bool{}, latest: map[string]usage.Record{}, @@ -110,25 +106,6 @@ func New(cfg Config) (*Reporter, error) { }, nil } -// checkURL accepts https, and http to a loopback host only. -func checkURL(rawURL string) error { - u, err := url.Parse(rawURL) - if err != nil { - return err - } - switch u.Scheme { - case "https": - case "http": - host := u.Hostname() - if ip := net.ParseIP(host); host != "localhost" && (ip == nil || !ip.IsLoopback()) { - return fmt.Errorf("http is allowed only for a loopback host, got %q", host) - } - default: - return fmt.Errorf("want an https URL, got scheme %q", u.Scheme) - } - return nil -} - // RunID returns the run named in r's totals. func (r *Reporter) RunID() string { return r.runID } @@ -198,7 +175,7 @@ func (r *Reporter) record(typ usage.Type, id, kid string) usage.Record { // timestamp is a record's ts: now, to the second. func (r *Reporter) timestamp() time.Time { - return r.now().UTC().Truncate(time.Second) + return time.Now().UTC().Truncate(time.Second) } // appendEvent queues an open or close, dropping the oldest past the cap. diff --git a/internal/meter/meter_test.go b/internal/meter/meter_test.go index 4100b766..4efe9e60 100644 --- a/internal/meter/meter_test.go +++ b/internal/meter/meter_test.go @@ -76,18 +76,12 @@ func bytesMetrics(in, out int64) map[string]int64 { return map[string]int64{usage.IngressBytes: in, usage.EgressBytes: out} } +// New takes a URL as reqsign.CheckURL does, which is tested there. func TestNew(t *testing.T) { - for rawURL, wantErr := range map[string]bool{ - "https://usage.example/v1": false, - "http://127.0.0.1:9000/v1": false, - "http://localhost/v1": false, - "http://usage.example/v1": true, - "ftp://usage.example/v1": true, - "https://usage.example/%zz": true, - } { - _, err := New(Config{URL: rawURL}) - assert.Equal(t, wantErr, err != nil, rawURL) - } + _, err := New(Config{URL: "https://usage.example/v1"}) + assert.NoError(t, err) + _, err = New(Config{URL: "http://usage.example/v1"}) + assert.Error(t, err, "a usage URL in the clear off loopback") } // A session's records go out in order: opens, usage, totals, closes, each diff --git a/internal/reqsign/reqsign.go b/internal/reqsign/reqsign.go index 6c271de0..58619728 100644 --- a/internal/reqsign/reqsign.go +++ b/internal/reqsign/reqsign.go @@ -37,7 +37,9 @@ import ( "errors" "fmt" "log/slog" + "net" "net/http" + "net/url" "strconv" "sync/atomic" "time" @@ -136,6 +138,27 @@ func signed(method, requestURI, timestamp string, body []byte) []byte { return []byte(requestContext + "\n" + method + "\n" + requestURI + "\n" + timestamp + "\n" + hex.EncodeToString(sum[:])) } +// CheckURL accepts the URL of a control-plane endpoint (a key set, a usage +// URL): https, or http only on a loopback host, so that the bearer token +// and the signature sent to it do not cross a network in the clear. +func CheckURL(rawURL string) error { + u, err := url.Parse(rawURL) + if err != nil { + return err + } + switch u.Scheme { + case "https": + case "http": + host := u.Hostname() + if ip := net.ParseIP(host); host != "localhost" && (ip == nil || !ip.IsLoopback()) { + return fmt.Errorf("http is allowed only for a loopback host, got %q", host) + } + default: + return fmt.Errorf("want an https URL, got scheme %q", u.Scheme) + } + return nil +} + // NoRedirectClient returns an HTTP client that follows no redirect: a 3xx is // answered like any other non-2xx. A client that sends signed requests must // use it: Go drops the bearer token on a redirect to another host but sends diff --git a/internal/reqsign/reqsign_test.go b/internal/reqsign/reqsign_test.go index 8d853595..84f7832a 100644 --- a/internal/reqsign/reqsign_test.go +++ b/internal/reqsign/reqsign_test.go @@ -257,3 +257,26 @@ func TestSigner_Rotate(t *testing.T) { }) } } +func TestCheckURL(t *testing.T) { + tests := map[string]struct { + url string + wantErrText string + }{ + "https": {url: "https://keys.example.com/v1/set"}, + "loopback http": {url: "http://127.0.0.1:4440/"}, + "localhost http": {url: "http://localhost:4440/"}, + "http off loopback": {url: "http://keys.example.com/", wantErrText: "loopback"}, + "another scheme": {url: "ftp://keys.example.com/", wantErrText: "https"}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + err := reqsign.CheckURL(tt.url) + + if tt.wantErrText != "" { + assert.ErrorContains(t, err, tt.wantErrText) + return + } + assert.NoError(t, err) + }) + } +} From 790d3f0df1e2f97037e9bcc847f7c77a31a67c92 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 11:15:24 +0000 Subject: [PATCH 4/7] feat(relay): report how long each publishing session kept broadcasts opted into HLS and ABR A publishing session's usage and session_close records now carry gateway.opt_in_seconds.abr and .hls (usage.OptInSeconds): the whole seconds its announcements beneath /.abr/ and /.hls/ lasted on the relay, added up over its broadcasts. Only an opt-in the session's grant covers reaches the relay, and moqt ends one a revoked ticket no longer covers, so the time is billed to the key that holds the opt-in. qumo abr never needs to know whose broadcast it transcodes. - relay: optInClock counts each checked session's opt-in announcements while they last; the lease sends its totals with each revalidate and reportEnd with the end (auth.Request.OptIns). - auth: a publisher's records carry them; a viewer's total does not. - optin.Of names the egress a path opts into; ingest uses it too. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KPWMJL7PH3rPhK5UTGavaB --- CHANGELOG.md | 1 + docs/site/content/en/docs/cli/abr.md | 5 + docs/site/content/en/docs/configuration.md | 5 +- internal/auth/auth.go | 7 ++ internal/auth/usage.go | 29 +++-- internal/auth/usage_viewers_test.go | 75 ++++++++++-- internal/auth/verifier.go | 8 +- internal/auth/verifier_test.go | 26 ++-- internal/ingest/auth.go | 13 +- internal/ingest/cmd.go | 4 +- internal/integration/optin_usage_test.go | 135 +++++++++++++++++++++ internal/optin/optin.go | 14 +++ internal/optin/optin_test.go | 18 +++ internal/relay/admit.go | 6 +- internal/relay/admit_test.go | 4 +- internal/relay/lease.go | 9 +- internal/relay/lease_credential_test.go | 2 +- internal/relay/lease_test.go | 16 +-- internal/relay/optin_time.go | 91 ++++++++++++++ internal/relay/optin_time_test.go | 77 ++++++++++++ internal/relay/server.go | 12 +- usage/usage.go | 14 +++ usage/usage_test.go | 5 + 23 files changed, 506 insertions(+), 70 deletions(-) create mode 100644 internal/integration/optin_usage_test.go create mode 100644 internal/relay/optin_time.go create mode 100644 internal/relay/optin_time_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 0cdcf6c2..430cb6d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **The playground opts a broadcast into ABR, and plays ABR broadcasts adaptively (`playground`).** The publish board's ABR box announces the broadcast's opt-in beneath `/.abr/` beside it, as the HLS scenario does beneath `/.hls/`. A broadcast whose video tracks share an alternate group is played adaptively: the viewer switches renditions at group boundaries, stepping down when frames fall behind their media time to what the rate measured since then carries, and up one rendition after a calm stretch. A step up keeps the old rendition until its last group before the switch has arrived; a step down drops it at once. The subscribe board picks a rendition by hand or automatically, and shows the rendition playing, the receive rate and the lag. - **`PEERS` names are resolved again while the relay runs, so peers behind a name join and leave without a restart; an entry without a port is an SRV name (`_moqt._udp.hubs.example.com`) (`internal/relay`).** A `PEERS` entry that is neither `host:port` nor a name without a port now stops the relay at startup, and a host that doesn't resolve is retried instead of being dialed by name. - **The relay re-reads its certificates while it runs (`internal/relay`).** About once a minute it reads `CERT_FILE`/`KEY_FILE`, `PEER_CERT_FILE`/`PEER_KEY_FILE` and `CA_FILE` and compares them with what it holds; what changed is taken from the next TLS handshake on, so cert-manager, a Vault or step-ca agent, or any job that writes the files needs no restart. A public certificate is taken once it parses and matches its key; a peer certificate and the CA together, once they pass the startup checks (issued by the CA, the peering name, valid now). Anything else is logged, once per distinct error rather than at every read, and the certificates in use are kept; renewal tools should replace the files atomically (write, then rename). Sessions already established are untouched. The CA is now checked at each handshake on both sides: as the dialed relay (`GetConfigForClient`) and as the dialer (`VerifyConnection` against the current CA, with the same checks crypto/tls made). The relay's key-set and usage requests are signed with the renewed peer certificate (`reqsign.Signer.Rotate`), and `qumo dns` re-reads its `PEER_CERT_FILE`/`PEER_KEY_FILE` on the same one-minute cadence and signs its feed reads with a renewed pair (`internal/certreload`, shared by both). New metrics: `qumo_relay_cert_reloads_total{cert,result}` and `qumo_relay_cert_not_after_seconds{cert}`, and `qumo_dns_cert_reloads_total{cert,result}` and `qumo_dns_cert_not_after_seconds{cert}`; alert on the `not_after` gauges. A CA is rotated by putting the new one beside the old in `CA_FILE` first ([TLS & mTLS](docs/site/content/en/docs/deployment/tls.md#certificate-renewal)). +- **A relay reports how long each publishing session kept broadcasts opted into HLS and ABR (`internal/relay`, `usage.OptInSeconds`).** A publishing session's `usage` and `session_close` records now carry `gateway.opt_in_seconds.abr` and `gateway.opt_in_seconds.hls`: the whole seconds its announcements beneath `/.abr/` and `/.hls/` lasted, added up over its broadcasts, from when the relay received each until it ended, the session closed or a revoked ticket ended it. Only an opt-in the session's grant covers reaches the relay, so the time is billed to the key that holds the opt-in, and a control plane can charge for ABR or HLS by the time it was used. `qumo abr` itself never needs to know whose broadcast it transcodes. A session that opted nothing into an egress carries no metric for it, so a receiver that doesn't bill opt-ins sees records as before. ### Changed diff --git a/docs/site/content/en/docs/cli/abr.md b/docs/site/content/en/docs/cli/abr.md index d98a20c4..6091d845 100644 --- a/docs/site/content/en/docs/cli/abr.md +++ b/docs/site/content/en/docs/cli/abr.md @@ -43,6 +43,11 @@ is refused within half a minute, and one on air ends at the next 30 s check of live sessions. HLS is held the same way (`"hls"`), and a key whose entry lists no `optIns` holds none. +A relay with `QUMO_USAGE_URL` reports how long each opt-in lasted, in the +publishing session's usage records under the tenant's key +(`gateway.opt_in_seconds.abr`, see [usage reports]({{< relref "../configuration#usage-reports" >}})), +so the control plane can bill ABR by the time it was used. + The tenant's app signs the opt-in's token with the same key as the broadcast's: `/.abr/acme/live/cam1` beside `/acme/live/cam1`, presented on the publisher's session as a ticket, since a session's credentials all come from diff --git a/docs/site/content/en/docs/configuration.md b/docs/site/content/en/docs/configuration.md index 29dbfffb..5a8dfed3 100644 --- a/docs/site/content/en/docs/configuration.md +++ b/docs/site/content/en/docs/configuration.md @@ -255,7 +255,8 @@ With `QUMO_USAGE_URL` set, the relay POSTs a JSON array of records to it every 1 {"type": "usage", "session_id": "…", "kid": "…", "metrics": {"gateway.ingress_bytes": 10162, "gateway.egress_bytes": 9223}, "ts": "…"}, {"type": "session_close", "session_id": "…", "kid": "…", - "metrics": {"gateway.ingress_bytes": 10438, "gateway.egress_bytes": 9165}, "reason": "closed", "ts": "…"}, + "metrics": {"gateway.ingress_bytes": 10438, "gateway.egress_bytes": 9165, + "gateway.opt_in_seconds.abr": 312}, "reason": "closed", "ts": "…"}, {"type": "viewers", "run": "…", "kid": "…", "metrics": {"gateway.ingress_bytes": 120, "gateway.egress_bytes": 880412}, "ts": "…"}] ``` @@ -264,6 +265,8 @@ Byte counts are cumulative and from the relay's point of view (what it received **Sessions that publish a broadcast are reported one by one,** as the first three records: an open, usage every 30 s, and a close. A session publishes when its token has a `publish` pattern, or from when it is granted a ticket that has one: its open comes then, and what it moved as a viewer up to its last 30 s check stays in its key's viewer total, the rest counting as its usage. +**A publishing session's records also carry its opt-in time:** how long it kept broadcasts opted into each egress, `gateway.opt_in_seconds.abr` for [`qumo abr`](../cli/abr/) and `gateway.opt_in_seconds.hls` for [`qumo hls`](../cli/hls/), in whole seconds, added up over its broadcasts (two broadcasts opted into ABR for a minute each are 120). It is the time each announcement of the session beneath `/.abr/` or `/.hls/` lasted on this relay, from when the relay received it until it ended, the session closed or a revoked ticket ended it, so a control plane can bill an opt-in by the time it was used. Only an opt-in the session's grant covers reaches the relay, so an opt-in a key doesn't hold is never counted. An egress the session opted nothing into has no metric. A broadcast opted in through [RTMP or RTSP ingest](../cli/rtmp/#opting-streams-into-hls-and-abr) counts on the ingest's session, not the publisher's. + **Every other session is a viewer, and viewers are reported together.** Viewers outnumber publishers by orders of magnitude, and a record per viewer would make the receiver's load grow with the audience. Their bytes are added up per key into one running total since the relay started, sent as a `viewers` record naming the key and the relay's `run`. The `run` is new each time the relay starts, so a restart begins new totals rather than lowering old ones. There is no open or close for viewers, and a total is sent only when it has changed. Records go out in batches of at most 500. A failed send, a 401, 403, 408, 413 or 429 included, keeps the records and retries them (logged once, then again when it recovers); only a batch the receiver can't read (400 or 422) is dropped. When the relay shuts down, it waits up to 5 s for its last sessions to record their end before the final send. diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 010991cb..1ba0df66 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -13,6 +13,9 @@ import ( "errors" "fmt" "net/http" + "time" + + "github.com/qumo-dev/qumo/token" ) // maxBody bounds how much of a reply's body is drained. @@ -73,6 +76,10 @@ type Request struct { // Bytes is the session's cumulative byte totals, set on revalidate and // end. Bytes Bytes + // OptIns is how long the session has kept broadcasts opted into each + // egress, added up over its broadcasts, set on revalidate and end + // (usage.OptInSeconds). An egress it opted nothing into is absent. + OptIns map[token.OptIn]time.Duration // Reason is why the session ended, set on end. Reason string } diff --git a/internal/auth/usage.go b/internal/auth/usage.go index 3832ff0d..7c54638b 100644 --- a/internal/auth/usage.go +++ b/internal/auth/usage.go @@ -8,6 +8,7 @@ import ( "github.com/qumo-dev/qumo/internal/meter" "github.com/qumo-dev/qumo/internal/reqsign" + "github.com/qumo-dev/qumo/token" "github.com/qumo-dev/qumo/usage" ) @@ -103,10 +104,10 @@ func (r *usageReporter) publishing(id string) { r.meter.Open(id, s.kid) } -// revalidated records a session's cumulative bytes at a revalidate and, when -// live is set, that the session is still live. A session it doesn't know is -// left unknown. -func (r *usageReporter) revalidated(id string, b Bytes, live bool) { +// revalidated records a session's cumulative bytes and opt-in time at a +// revalidate and, when live is set, that the session is still live. A +// session it doesn't know is left unknown. +func (r *usageReporter) revalidated(id string, b Bytes, optIns map[token.OptIn]time.Duration, live bool) { r.mu.Lock() defer r.mu.Unlock() s, ok := r.sessions[id] @@ -119,15 +120,15 @@ func (r *usageReporter) revalidated(id string, b Bytes, live bool) { switch { case !s.publishes: s = r.countViewer(s, b) - case b.Sent > s.counted.Sent || b.Received > s.counted.Received: - r.meter.Report(id, s.kid, usageMetrics(b.since(s.counted))) + case b.Sent > s.counted.Sent || b.Received > s.counted.Received || len(optIns) > 0: + r.meter.Report(id, s.kid, publisherMetrics(b.since(s.counted), optIns)) } r.sessions[id] = s } // close records a session's end with its final totals and forgets it. It // reports false for a session it doesn't know. -func (r *usageReporter) close(id string, b Bytes, reason string) bool { +func (r *usageReporter) close(id string, b Bytes, optIns map[token.OptIn]time.Duration, reason string) bool { r.mu.Lock() defer r.mu.Unlock() r.closed[id] = r.now() @@ -140,7 +141,7 @@ func (r *usageReporter) close(id string, b Bytes, reason string) bool { r.countViewer(s, b) return true } - r.meter.Close(id, s.kid, usageMetrics(b.since(s.counted)), reason) + r.meter.Close(id, s.kid, publisherMetrics(b.since(s.counted), optIns), reason) return true } @@ -166,6 +167,18 @@ func usageMetrics(b Bytes) map[string]int64 { return map[string]int64{usage.IngressBytes: clampInt64(b.Received), usage.EgressBytes: clampInt64(b.Sent)} } +// publisherMetrics is a publishing session's metrics: its bytes, and how +// long it kept broadcasts opted into each egress, in whole seconds. +func publisherMetrics(b Bytes, optIns map[token.OptIn]time.Duration) map[string]int64 { + m := usageMetrics(b) + for o, d := range optIns { + if secs := int64(d / time.Second); secs > 0 { + m[usage.OptInSeconds(string(o))] = secs + } + } + return m +} + func clampInt64(v uint64) int64 { if v > math.MaxInt64 { return math.MaxInt64 diff --git a/internal/auth/usage_viewers_test.go b/internal/auth/usage_viewers_test.go index 79ee16e7..5775422a 100644 --- a/internal/auth/usage_viewers_test.go +++ b/internal/auth/usage_viewers_test.go @@ -4,7 +4,9 @@ import ( "context" "net/http" "testing" + "time" + "github.com/qumo-dev/qumo/token" "github.com/qumo-dev/qumo/usage" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -25,9 +27,9 @@ func TestUsageReporter_ViewersAreTotalled(t *testing.T) { r.open(id, viewerSession("k1")) } r.open("other", viewerSession("k2")) - r.revalidated("v1", Bytes{Sent: 100, Received: 10}, true) - r.revalidated("v2", Bytes{Sent: 200, Received: 20}, true) - r.revalidated("other", Bytes{Sent: 7}, true) + r.revalidated("v1", Bytes{Sent: 100, Received: 10}, nil, true) + r.revalidated("v2", Bytes{Sent: 200, Received: 20}, nil, true) + r.revalidated("other", Bytes{Sent: 7}, nil, true) require.NoError(t, r.meter.Flush(context.Background())) first := sink.received()[0] @@ -49,11 +51,11 @@ func TestUsageReporter_ViewersAreTotalled(t *testing.T) { }) t.Run("the total grows by what each session moved since it was last counted", func(t *testing.T) { - r.revalidated("v1", Bytes{Sent: 150, Received: 10}, true) // +50 - r.revalidated("v1", Bytes{Sent: 120, Received: 10}, true) // a late, lower report adds nothing - assert.True(t, r.close("v2", Bytes{Sent: 260, Received: 25}, "closed")) // +60, +5 - r.revalidated("v2", Bytes{Sent: 999}, true) // after its close: ignored - assert.True(t, r.close("v3", Bytes{}, "closed")) // moved nothing + r.revalidated("v1", Bytes{Sent: 150, Received: 10}, nil, true) // +50 + r.revalidated("v1", Bytes{Sent: 120, Received: 10}, nil, true) // a late, lower report adds nothing + assert.True(t, r.close("v2", Bytes{Sent: 260, Received: 25}, nil, "closed")) // +60, +5 + r.revalidated("v2", Bytes{Sent: 999}, nil, true) // after its close: ignored + assert.True(t, r.close("v3", Bytes{}, nil, "closed")) // moved nothing require.NoError(t, r.meter.Flush(context.Background())) batches := sink.received() @@ -67,10 +69,10 @@ func TestUsageReporter_ViewerTotalSurvivesAFailedSend(t *testing.T) { sink := &fakeUsageSink{statuses: []int{http.StatusServiceUnavailable, http.StatusOK}} r := newTestUsage(t, sink) r.open("v1", viewerSession("k1")) - r.revalidated("v1", Bytes{Sent: 100}, true) + r.revalidated("v1", Bytes{Sent: 100}, nil, true) assert.Error(t, r.meter.Flush(context.Background())) - r.revalidated("v1", Bytes{Sent: 160}, true) + r.revalidated("v1", Bytes{Sent: 160}, nil, true) require.NoError(t, r.meter.Flush(context.Background())) batches := sink.received() @@ -84,7 +86,7 @@ func TestUsageReporter_PublishersStayPerSession(t *testing.T) { r := newTestUsage(t, sink) r.open("p1", usageSession{kid: "k1", publishes: true}) r.open("p2", usageSession{kid: "k1", publishes: true}) - r.revalidated("p1", Bytes{Received: 50}, true) + r.revalidated("p1", Bytes{Received: 50}, nil, true) require.NoError(t, r.meter.Flush(context.Background())) var got []string @@ -107,7 +109,7 @@ func TestUsageReporter_Publishing(t *testing.T) { r.publishing("v1") r.publishing("p1") r.publishing("unknown") - r.revalidated("v1", Bytes{Received: 40}, true) + r.revalidated("v1", Bytes{Received: 40}, nil, true) require.NoError(t, r.meter.Flush(context.Background())) var got []string @@ -132,3 +134,52 @@ func TestBytes_Since(t *testing.T) { }) } } + +// A publishing session's usage carries how long it kept broadcasts opted +// into each egress, in whole seconds, and is reported for it even when the +// session moved no bytes; its close carries the final time. A viewer's +// total carries none. +func TestUsageReporter_OptInSeconds(t *testing.T) { + sink := &fakeUsageSink{} + r := newTestUsage(t, sink) + r.open("p1", usageSession{kid: "k1", publishes: true}) + r.open("v1", viewerSession("k1")) + abr := func(d time.Duration) map[token.OptIn]time.Duration { + return map[token.OptIn]time.Duration{token.OptInABR: d} + } + + r.revalidated("p1", Bytes{Received: 100}, abr(1500*time.Millisecond), true) + r.revalidated("v1", Bytes{Sent: 10}, abr(time.Hour), true) + require.NoError(t, r.meter.Flush(context.Background())) + first := byType(sink.received()[0]) + assert.Equal(t, map[string]int64{ + usage.IngressBytes: 100, usage.EgressBytes: 0, usage.OptInSeconds("abr"): 1, + }, first[usage.Usage].Metrics, "whole seconds") + assert.NotContains(t, first[usage.Viewers].Metrics, usage.OptInSeconds("abr"), "a viewer opts nothing in") + + r.open("p2", usageSession{kid: "k1", publishes: true}) + r.revalidated("p2", Bytes{}, abr(2*time.Second), true) + require.NoError(t, r.meter.Flush(context.Background())) + require.Len(t, sink.received(), 2) + assert.Equal(t, int64(2), byType(sink.received()[1])[usage.Usage].Metrics[usage.OptInSeconds("abr")], + "opt-in time without bytes is reported") + + assert.True(t, r.close("p1", Bytes{Received: 120}, map[token.OptIn]time.Duration{ + token.OptInABR: 3 * time.Second, token.OptInHLS: 500 * time.Millisecond, + }, usage.ReasonClosed)) + require.NoError(t, r.meter.Flush(context.Background())) + closed := byType(sink.received()[2])[usage.SessionClose] + assert.Equal(t, map[string]int64{ + usage.IngressBytes: 120, usage.EgressBytes: 0, usage.OptInSeconds("abr"): 3, + }, closed.Metrics, "under a second of HLS is left out") + assert.NoError(t, closed.Validate()) +} + +// byType returns batch's records by type, the last of each. +func byType(batch []usage.Record) map[usage.Type]usage.Record { + out := map[usage.Type]usage.Record{} + for _, rec := range batch { + out[rec.Type] = rec + } + return out +} diff --git a/internal/auth/verifier.go b/internal/auth/verifier.go index 0d29e9db..b7726521 100644 --- a/internal/auth/verifier.go +++ b/internal/auth/verifier.go @@ -146,7 +146,7 @@ func (v *Verifier) Authorize(_ context.Context, req Request) (*Grant, error) { // A refused session is ending; one that couldn't be checked // lives on, and the relay asks again. _, refused := errors.AsType[RefusedError](err) - v.usage.revalidated(req.ID, req.Bytes, !refused) + v.usage.revalidated(req.ID, req.Bytes, req.OptIns, !refused) } } return g, err @@ -157,7 +157,7 @@ func (v *Verifier) Authorize(_ context.Context, req Request) (*Grant, error) { // counted once a ticket binds it to a key (Bind). func (v *Verifier) anonymous(req Request) *Grant { if v.usage != nil && req.Event == EventRevalidate { - v.usage.revalidated(req.ID, req.Bytes, true) + v.usage.revalidated(req.ID, req.Bytes, req.OptIns, true) } return &Grant{revalidate: revalidateEvery, anonymous: true} } @@ -302,11 +302,11 @@ func publishesBroadcast(g token.Grant) bool { return len(g.Publish) > 0 } -// End reports a session's end with its final byte totals. +// End reports a session's end with its final byte totals and opt-in time. func (v *Verifier) End(_ context.Context, req Request) error { if v.usage == nil { return nil } - v.usage.close(req.ID, req.Bytes, req.Reason) + v.usage.close(req.ID, req.Bytes, req.OptIns, req.Reason) return nil } diff --git a/internal/auth/verifier_test.go b/internal/auth/verifier_test.go index 605f2d2e..c7782481 100644 --- a/internal/auth/verifier_test.go +++ b/internal/auth/verifier_test.go @@ -405,8 +405,8 @@ func TestUsageReporter(t *testing.T) { sink := &fakeUsageSink{} r := newTestUsage(t, sink) r.open("s1", testUsageSession) - r.revalidated("s1", Bytes{Sent: 1, Received: 10}, true) - r.revalidated("s1", Bytes{Sent: 2, Received: 20}, true) + r.revalidated("s1", Bytes{Sent: 1, Received: 10}, nil, true) + r.revalidated("s1", Bytes{Sent: 2, Received: 20}, nil, true) require.NoError(t, r.meter.Flush(context.Background())) batches := sink.received() require.Len(t, batches, 1) @@ -419,9 +419,9 @@ func TestUsageReporter(t *testing.T) { sink := &fakeUsageSink{statuses: []int{http.StatusBadGateway, http.StatusOK}} r := newTestUsage(t, sink) r.open("s1", testUsageSession) - r.revalidated("s1", Bytes{Received: 10}, true) + r.revalidated("s1", Bytes{Received: 10}, nil, true) assert.Error(t, r.meter.Flush(context.Background())) - r.revalidated("s1", Bytes{Received: 20}, true) + r.revalidated("s1", Bytes{Received: 20}, nil, true) require.NoError(t, r.meter.Flush(context.Background())) retry := sink.received()[1] require.Len(t, retry, 2) @@ -435,8 +435,8 @@ func TestUsageReporter(t *testing.T) { t.Run("an unknown session's end and usage are ignored", func(t *testing.T) { sink := &fakeUsageSink{} r := newTestUsage(t, sink) - assert.False(t, r.close("nope", Bytes{Sent: 1}, "")) - r.revalidated("nope", Bytes{Sent: 1}, true) + assert.False(t, r.close("nope", Bytes{Sent: 1}, nil, "")) + r.revalidated("nope", Bytes{Sent: 1}, nil, true) require.NoError(t, r.meter.Flush(context.Background())) assert.Empty(t, sink.received(), "nothing pending, nothing sent") }) @@ -450,13 +450,13 @@ func TestUsageReporter(t *testing.T) { r.open("live", testUsageSession) now = now.Add(sessionGrace + time.Second) - r.revalidated("refused", Bytes{Sent: 1}, false) - r.revalidated("live", Bytes{}, true) + r.revalidated("refused", Bytes{Sent: 1}, nil, false) + r.revalidated("live", Bytes{}, nil, true) r.forgetStale() - assert.False(t, r.close("gone", Bytes{}, ""), "not revalidated within the grace") - assert.False(t, r.close("refused", Bytes{}, ""), "a refused revalidate doesn't keep it") - assert.True(t, r.close("live", Bytes{}, ""), "revalidated: still remembered") + assert.False(t, r.close("gone", Bytes{}, nil, ""), "not revalidated within the grace") + assert.False(t, r.close("refused", Bytes{}, nil, ""), "a refused revalidate doesn't keep it") + assert.True(t, r.close("live", Bytes{}, nil, ""), "revalidated: still remembered") }) } @@ -556,7 +556,7 @@ func TestVerifier_RevalidateAfterEnd(t *testing.T) { types = append(types, rec.Type) } assert.Equal(t, []usage.Type{usage.SessionOpen, usage.SessionClose}, types) - assert.False(t, v.usage.close(late.ID, Bytes{}, ""), "the session isn't known again") + assert.False(t, v.usage.close(late.ID, Bytes{}, nil, ""), "the session isn't known again") } // writeKeySet writes a key set file holding keys and returns its path. @@ -972,7 +972,7 @@ func TestUsageReporter_OpenAfterClose(t *testing.T) { now := time.Now() r.now = func() time.Time { return now } - r.close("s1", Bytes{}, "closed") + r.close("s1", Bytes{}, nil, "closed") r.open("s1", usageSession{kid: "k1", publishes: true}) r.publishing("s1") diff --git a/internal/ingest/auth.go b/internal/ingest/auth.go index 239fbcb2..4e9ab881 100644 --- a/internal/ingest/auth.go +++ b/internal/ingest/auth.go @@ -52,21 +52,10 @@ func rtmpTarget(app, streamKey string) (moqt.BroadcastPath, []string) { // and media there would answer an egress that expects a redirect. var errOptInPath = errors.New("the path lies beneath an opt-in directory, where the server opts streams in: push to the broadcast's own path") -// optInDirectory returns the opt-in directory path lies beneath, such as -// /.abr/ for /.abr/live/cam, and false when it lies beneath none. -func optInDirectory(path moqt.BroadcastPath) (string, bool) { - for _, o := range token.OptIns() { - if dir := o.Directory(); strings.HasPrefix(string(path), dir) { - return dir, true - } - } - return "", false -} - // checkPushPath refuses a push to path when the server keeps it for opt-ins // (errOptInPath). func checkPushPath(path moqt.BroadcastPath) error { - if _, ok := optInDirectory(path); ok { + if _, ok := optin.Of(path); ok { return fmt.Errorf("%s: %w", path, errOptInPath) } return nil diff --git a/internal/ingest/cmd.go b/internal/ingest/cmd.go index 80de060b..87583c9f 100644 --- a/internal/ingest/cmd.go +++ b/internal/ingest/cmd.go @@ -290,8 +290,8 @@ func publishAuthorizer(v *auth.Verifier, beneath string) PublishAuthorizer { } return func(credential string, path moqt.BroadcastPath) (time.Time, error) { broadcast := path - if dir, ok := optInDirectory(path); ok { - b, named := optin.Broadcast(string(path), dir) + if o, ok := optin.Of(path); ok { + b, named := optin.Broadcast(string(path), o.Directory()) if !named { // Cleaned, it could name a broadcast outside the // directory the relay grants its exact path in. diff --git a/internal/integration/optin_usage_test.go b/internal/integration/optin_usage_test.go new file mode 100644 index 00000000..ce5f3571 --- /dev/null +++ b/internal/integration/optin_usage_test.go @@ -0,0 +1,135 @@ +//go:build integration + +package integration + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/qumo-dev/gomoqt/moqt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/qumo-dev/qumo/internal/auth" + "github.com/qumo-dev/qumo/internal/optin" + "github.com/qumo-dev/qumo/internal/relay" + "github.com/qumo-dev/qumo/token" + "github.com/qumo-dev/qumo/usage" +) + +// usageSink records the usage records POSTed to it. +type usageSink struct { + mu sync.Mutex + records []usage.Record +} + +func (s *usageSink) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var batch []usage.Record + if err := json.NewDecoder(r.Body).Decode(&batch); err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.records = append(s.records, batch...) +} + +func (s *usageSink) all() []usage.Record { + s.mu.Lock() + defer s.mu.Unlock() + return append([]usage.Record(nil), s.records...) +} + +// TestRelay_ReportsOptInSeconds runs a paid tenant's publisher against a +// relay that reports usage: it publishes a broadcast and, on a ticket its key +// holds the ABR opt-in for, opts it into ABR. The close record of its session +// carries how long the opt-in lasted (usage.OptInSeconds), under the +// tenant's key, so a control plane can bill ABR by the time it was used. +func TestRelay_ReportsOptInSeconds(t *testing.T) { + const src = moqt.BroadcastPath("/live/metered") + optInPath := optin.Path(token.ABRDirectory, src) + + tenant, err := token.GenerateKey("/live/") + require.NoError(t, err) + tenantPub := tenant.Public() + tenantPub.OptIns = []token.OptIn{token.OptInABR} + set, err := token.MarshalKeySet(tenantPub) + require.NoError(t, err) + keys := filepath.Join(t.TempDir(), "keys.json") + require.NoError(t, os.WriteFile(keys, set, 0o600)) + sink := &usageSink{} + usageSrv := httptest.NewServer(sink) + t.Cleanup(usageSrv.Close) + verifier, err := auth.NewVerifier(auth.VerifierConfig{Keys: keys, UsageURL: usageSrv.URL}) + require.NoError(t, err) + runCtx, stopRun := context.WithCancel(context.Background()) + runDone := make(chan struct{}) + go func() { + defer close(runDone) + verifier.Run(runCtx) + }() + t.Cleanup(func() { stopRun(); <-runDone }) + + ended := make(chan auth.Request, 4) + addr, srv := startAuthRelay(t, verifier.Authorize, nil, func(s *relay.Server) { + s.Tickets = verifier + s.End = func(ctx context.Context, req auth.Request) error { + err := verifier.End(ctx, req) + ended <- req + return err + } + }) + sign := func(g token.Grant) string { + tok, err := token.Sign(tenant, g, time.Minute) + require.NoError(t, err) + return tok + } + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + pubMux := moqt.NewTrackMux(0) + pubMux.PublishFunc(ctx, src, func(tw *moqt.TrackWriter) { <-tw.Context().Done() }) + // Native QUIC: see TestABR_PaidOptIn. + pub := dialOver(t, nativeURL(addr)+"/?jwt="+sign(token.Grant{Root: "/live/", Publish: []string{"/metered"}}), nil, pubMux) + actx, acancel := context.WithTimeout(ctx, 3*time.Second) + defer acancel() + _, err = pub.Auth(actx, []byte(sign(token.Grant{Root: "/.abr/live/", Publish: []string{"/metered"}}))) + require.NoError(t, err, "the key holds ABR") + pubMux.Publish(ctx, optInPath, optin.Handler(src)) + require.Eventually(t, routed(srv, optInPath), 5*time.Second, 25*time.Millisecond) + + // Opted in for a little over two seconds, then the publisher leaves. + time.Sleep(2200 * time.Millisecond) + require.NoError(t, pub.CloseWithError(moqt.NoError, "done")) + var end auth.Request + select { + case end = <-ended: + case <-time.After(5 * time.Second): + t.Fatal("the relay did not report the session's end") + } + assert.GreaterOrEqual(t, end.OptIns[token.OptInABR], 2*time.Second) + assert.NotContains(t, end.OptIns, token.OptInHLS, "nothing was opted into HLS") + + // The verifier sends what is pending once more when it stops. + stopRun() + <-runDone + var closed *usage.Record + for _, rec := range sink.all() { + if rec.Type == usage.SessionClose && rec.SessionID == end.ID { + closed = &rec + } + } + require.NotNil(t, closed, "no session_close for the publisher's session in %v", sink.all()) + assert.Equal(t, tenant.ID, closed.KID) + secs := closed.Metrics[usage.OptInSeconds("abr")] + assert.GreaterOrEqual(t, secs, int64(2)) + assert.LessOrEqual(t, secs, int64(5), "the time the opt-in lasted, not longer") + assert.NoError(t, closed.Validate()) +} diff --git a/internal/optin/optin.go b/internal/optin/optin.go index ae8b3241..a2db4d88 100644 --- a/internal/optin/optin.go +++ b/internal/optin/optin.go @@ -11,6 +11,8 @@ import ( "strings" "github.com/qumo-dev/gomoqt/moqt" + + "github.com/qumo-dev/qumo/token" ) // Handler serves the opt-in of broadcast, announced as broadcast's path @@ -25,6 +27,18 @@ func Handler(broadcast moqt.BroadcastPath) moqt.TrackHandler { }) } +// Of returns the egress p opts a broadcast into, when p lies beneath an +// opt-in directory: token.OptInABR for /.abr/live/cam1. It reports false for +// a path beneath none. +func Of(p moqt.BroadcastPath) (token.OptIn, bool) { + for _, o := range token.OptIns() { + if strings.HasPrefix(string(p), o.Directory()) { + return o, true + } + } + return "", false +} + // Path returns the opt-in of broadcast beneath dir, an opt-in directory: // Path("/.abr/", "/live/cam1") is "/.abr/live/cam1". func Path(dir string, broadcast moqt.BroadcastPath) moqt.BroadcastPath { diff --git a/internal/optin/optin_test.go b/internal/optin/optin_test.go index d63c69bd..2c7ae399 100644 --- a/internal/optin/optin_test.go +++ b/internal/optin/optin_test.go @@ -42,6 +42,24 @@ func TestBroadcast(t *testing.T) { } } +func TestOf(t *testing.T) { + tests := map[moqt.BroadcastPath]struct { + want token.OptIn + wantOK bool + }{ + "/.abr/live/cam1": {want: token.OptInABR, wantOK: true}, + "/.hls/live/cam1": {want: token.OptInHLS, wantOK: true}, + "/live/cam1": {}, + "/.abrx/cam1": {}, + "/live/.abr/cam1": {}, + } + for p, tt := range tests { + got, ok := Of(p) + assert.Equal(t, tt.wantOK, ok, p) + assert.Equal(t, tt.want, got, p) + } +} + func TestHasDotSegment(t *testing.T) { tests := map[string]struct { path string diff --git a/internal/relay/admit.go b/internal/relay/admit.go index 2627b4d1..4030d9f8 100644 --- a/internal/relay/admit.go +++ b/internal/relay/admit.go @@ -698,15 +698,17 @@ func (s *Server) admitClient(ctx context.Context, req auth.Request) (g *auth.Gra } // reportEnd sends the end event of a checked session admitted by req, with -// its final byte totals and why it ended. ctx is the session's: done by now, +// its final byte totals, its opt-in time (optIns, nil for none) and why it +// ended. ctx is the session's: done by now, // so the report keeps its values but not its cancellation. A failed report is // recorded and otherwise ignored; End bounds its own wait. -func (s *Server) reportEnd(ctx context.Context, req auth.Request, stats moqt.SessionStats, reason string) { +func (s *Server) reportEnd(ctx context.Context, req auth.Request, stats moqt.SessionStats, optIns *optInClock, reason string) { if s.End == nil { return } req.Event = auth.EventEnd req.Bytes = sessionBytes(stats) + req.OptIns = optIns.totals() req.Reason = reason if err := s.End(context.WithoutCancel(ctx), req); err != nil { metricAuthRequests.WithLabelValues(auth.EventEnd, authError).Inc() diff --git a/internal/relay/admit_test.go b/internal/relay/admit_test.go index 9711b688..13c19656 100644 --- a/internal/relay/admit_test.go +++ b/internal/relay/admit_test.go @@ -193,7 +193,7 @@ func TestServer_ReportEnd(t *testing.T) { connect := auth.Request{ID: "00ff", Event: auth.EventConnect, Path: "/acme", Query: "jwt=h.p.s"} delta := counterDelta(t, func() { - srv.reportEnd(context.Background(), connect, stats, endDropped) + srv.reportEnd(context.Background(), connect, stats, nil, endDropped) }, metricAuthRequests.WithLabelValues(auth.EventEnd, tt.wantResult)) assert.Equal(t, 1.0, delta) @@ -210,7 +210,7 @@ func TestServer_ReportEnd(t *testing.T) { // reports nothing. func TestServer_ReportEnd_Off(t *testing.T) { delta := counterDelta(t, func() { - (&Server{}).reportEnd(context.Background(), auth.Request{ID: "00ff"}, moqt.SessionStats{}, endClosed) + (&Server{}).reportEnd(context.Background(), auth.Request{ID: "00ff"}, moqt.SessionStats{}, nil, endClosed) }, metricAuthRequests.WithLabelValues(auth.EventEnd, authOK)) assert.Zero(t, delta) diff --git a/internal/relay/lease.go b/internal/relay/lease.go index ed2166d4..53d0ae12 100644 --- a/internal/relay/lease.go +++ b/internal/relay/lease.go @@ -59,6 +59,9 @@ type lease struct { sess leasedSession authorize authorizeFunc req auth.Request + // optIns is the session's opt-in time, sent with each revalidate; nil + // for none. + optIns *optInClock mu sync.Mutex timer *time.Timer @@ -72,9 +75,9 @@ type lease struct { } // startLease starts the lease of a session admitted by req, to be revalidated -// every cadence. It returns nil for a zero cadence, which never revalidates; +// every cadence, reporting the opt-in time optIns counts (nil for none). It returns nil for a zero cadence, which never revalidates; // otherwise the caller stops it when the session ends first. -func startLease(ctx context.Context, sess leasedSession, authorize authorizeFunc, req auth.Request, cadence time.Duration) *lease { +func startLease(ctx context.Context, sess leasedSession, authorize authorizeFunc, req auth.Request, cadence time.Duration, optIns *optInClock) *lease { if cadence <= 0 { return nil } @@ -83,6 +86,7 @@ func startLease(ctx context.Context, sess leasedSession, authorize authorizeFunc sess: sess, authorize: authorize, req: req, + optIns: optIns, cadence: cadence, } l.mu.Lock() @@ -167,6 +171,7 @@ func (l *lease) revalidate() (*auth.Grant, error) { req := l.req req.Event = auth.EventRevalidate req.Bytes = sessionBytes(l.sess.Stats()) + req.OptIns = l.optIns.totals() g, err := l.authorize(l.ctx, req) metricAuthRequests.WithLabelValues(auth.EventRevalidate, authOutcome(g, err)).Inc() return g, err diff --git a/internal/relay/lease_credential_test.go b/internal/relay/lease_credential_test.go index d88ba27b..73b303ad 100644 --- a/internal/relay/lease_credential_test.go +++ b/internal/relay/lease_credential_test.go @@ -42,7 +42,7 @@ func verifiedSession(t *testing.T, v *auth.Verifier, key token.SigningKey) (*fak g, err := v.Authorize(t.Context(), req) require.NoError(t, err) sess := &fakeLeasedSession{} - l := startLease(t.Context(), sess, v.Authorize, req, g.Revalidate()) + l := startLease(t.Context(), sess, v.Authorize, req, g.Revalidate(), nil) require.NotNil(t, l, "every verified session is re-checked") return sess, l } diff --git a/internal/relay/lease_test.go b/internal/relay/lease_test.go index 7fbd7c43..668fd201 100644 --- a/internal/relay/lease_test.go +++ b/internal/relay/lease_test.go @@ -22,7 +22,7 @@ import ( var leaseRequest = auth.Request{ID: "00ff", Event: auth.EventConnect, Path: "/acme", Query: "jwt=h.p.s"} func TestStartLease_NothingToDo(t *testing.T) { - assert.Nil(t, startLease(context.Background(), &fakeLeasedSession{}, (&fakeAuth{}).authorize, leaseRequest, 0)) + assert.Nil(t, startLease(context.Background(), &fakeLeasedSession{}, (&fakeAuth{}).authorize, leaseRequest, 0, nil)) } // TestLease_RevalidateEnds verifies a refused revalidate ends @@ -41,7 +41,7 @@ func TestLease_RevalidateEnds(t *testing.T) { server := &fakeAuth{err: tt.err} sess := &fakeLeasedSession{} sess.setStats(1500, 300) - l := startLease(t.Context(), sess, server.authorize, leaseRequest, 30*time.Second) + l := startLease(t.Context(), sess, server.authorize, leaseRequest, 30*time.Second, nil) defer l.stop() time.Sleep(30*time.Second - time.Nanosecond) @@ -67,7 +67,7 @@ func TestLease_RevalidateTakesCadence(t *testing.T) { // The reply's grant differs from connect's; only the cadence matters. server := &fakeAuth{grant: testGrant(t, "other/**", "", time.Minute)} sess := &fakeLeasedSession{} - l := startLease(t.Context(), sess, server.authorize, leaseRequest, 30*time.Second) + l := startLease(t.Context(), sess, server.authorize, leaseRequest, 30*time.Second, nil) defer l.stop() time.Sleep(150 * time.Second) @@ -89,7 +89,7 @@ func TestLease_RevalidateUnavailable(t *testing.T) { t.Run(name, func(t *testing.T) { synctest.Test(t, func(t *testing.T) { sess := &fakeLeasedSession{} - l := startLease(t.Context(), sess, server.authorize, leaseRequest, 30*time.Second) + l := startLease(t.Context(), sess, server.authorize, leaseRequest, 30*time.Second, nil) defer l.stop() time.Sleep(time.Hour) @@ -103,7 +103,7 @@ func TestLease_RevalidateUnavailable(t *testing.T) { t.Run("retries back off", func(t *testing.T) { synctest.Test(t, func(t *testing.T) { server := &fakeAuth{err: errors.New("503")} - l := startLease(t.Context(), &fakeLeasedSession{}, server.authorize, leaseRequest, 30*time.Second) + l := startLease(t.Context(), &fakeLeasedSession{}, server.authorize, leaseRequest, 30*time.Second, nil) defer l.stop() // Retries follow the first attempt at 30 s with jittered delays of @@ -123,7 +123,7 @@ func TestLease_Stop(t *testing.T) { synctest.Test(t, func(t *testing.T) { server := &fakeAuth{} sess := &fakeLeasedSession{} - l := startLease(t.Context(), sess, server.authorize, leaseRequest, 30*time.Second) + l := startLease(t.Context(), sess, server.authorize, leaseRequest, 30*time.Second, nil) time.Sleep(10 * time.Second) l.stop() @@ -162,7 +162,7 @@ func TestLease_RevalidateReportsBytes(t *testing.T) { synctest.Test(t, func(t *testing.T) { server := &fakeAuth{grant: testGrant(t, "", "**", 30*time.Second)} sess := &fakeLeasedSession{} - l := startLease(t.Context(), sess, server.authorize, leaseRequest, 30*time.Second) + l := startLease(t.Context(), sess, server.authorize, leaseRequest, 30*time.Second, nil) defer l.stop() sess.setStats(100, 10) @@ -219,7 +219,7 @@ func TestLease_RevalidateWithoutCadenceStops(t *testing.T) { synctest.Test(t, func(t *testing.T) { server := &fakeAuth{grant: testGrant(t, "", "**", 0)} sess := &fakeLeasedSession{} - l := startLease(t.Context(), sess, server.authorize, leaseRequest, 30*time.Second) + l := startLease(t.Context(), sess, server.authorize, leaseRequest, 30*time.Second, nil) defer l.stop() time.Sleep(time.Hour) diff --git a/internal/relay/optin_time.go b/internal/relay/optin_time.go new file mode 100644 index 00000000..568b87e2 --- /dev/null +++ b/internal/relay/optin_time.go @@ -0,0 +1,91 @@ +package relay + +import ( + "sync" + "time" + + "github.com/qumo-dev/gomoqt/moqt" + + "github.com/qumo-dev/qumo/internal/optin" + "github.com/qumo-dev/qumo/token" +) + +// optInClock adds up how long a session keeps broadcasts opted into each +// egress: how long each announcement of its beneath an opt-in directory +// (/.abr/live/cam) lasts, summed over its announcements, so two broadcasts +// opted into ABR for a minute each are two minutes. The session's usage +// reports carry it (auth.Request.OptIns), for a control plane to bill an +// opt-in by the time it was used. A nil clock counts nothing. +// +// It counts what the relay received from the session: an announcement the +// session's grant does not cover never reaches it, so an opt-in the +// session's key does not hold is never counted. +type optInClock struct { + now func() time.Time + + mu sync.Mutex + tallies map[token.OptIn]*optInTally +} + +// optInTally is one egress's time: done, from announcements that ended, +// plus live announcements since since, the last time live changed. +type optInTally struct { + done time.Duration + live int + since time.Time +} + +func newOptInClock(now func() time.Time) *optInClock { + return &optInClock{now: now, tallies: map[token.OptIn]*optInTally{}} +} + +// track counts ann, an announcement the session made, until it ends, when it +// opts a broadcast into an egress; any other is not counted, nor one that +// ended already (moqt runs AfterFunc for it later, on another goroutine, +// which would count the time until then). +func (c *optInClock) track(ann *moqt.Announcement) { + if c == nil || !ann.IsActive() { + return + } + o, ok := optin.Of(ann.BroadcastPath()) + if !ok { + return + } + c.change(o, +1) + ann.AfterFunc(func() { c.change(o, -1) }) +} + +// change adds delta to o's live announcements, counting the time they were +// live until now. +func (c *optInClock) change(o token.OptIn, delta int) { + c.mu.Lock() + defer c.mu.Unlock() + now := c.now() + t := c.tallies[o] + if t == nil { + t = &optInTally{} + c.tallies[o] = t + } + t.done += time.Duration(t.live) * now.Sub(t.since) + t.live += delta + t.since = now +} + +// totals returns how long the session has kept broadcasts opted into each +// egress so far, live announcements counted up to now, or nil for none. +func (c *optInClock) totals() map[token.OptIn]time.Duration { + if c == nil { + return nil + } + c.mu.Lock() + defer c.mu.Unlock() + if len(c.tallies) == 0 { + return nil + } + now := c.now() + out := make(map[token.OptIn]time.Duration, len(c.tallies)) + for o, t := range c.tallies { + out[o] = t.done + time.Duration(t.live)*now.Sub(t.since) + } + return out +} diff --git a/internal/relay/optin_time_test.go b/internal/relay/optin_time_test.go new file mode 100644 index 00000000..be1cda27 --- /dev/null +++ b/internal/relay/optin_time_test.go @@ -0,0 +1,77 @@ +package relay + +import ( + "context" + "testing" + "time" + + "github.com/qumo-dev/gomoqt/moqt" + "github.com/stretchr/testify/assert" + + "github.com/qumo-dev/qumo/token" +) + +// fakeClock is a time that moves only when told. +type fakeClock struct{ t time.Time } + +func (c *fakeClock) now() time.Time { return c.t } +func (c *fakeClock) add(d time.Duration) { c.t = c.t.Add(d) } + +// announce returns a live announcement of path and the function that ends it. +func announce(t *testing.T, path moqt.BroadcastPath) (*moqt.Announcement, func()) { + t.Helper() + ann, end := moqt.NewAnnouncement(context.Background(), path) + t.Cleanup(end) + return ann, end +} + +func TestOptInClock(t *testing.T) { + clock := &fakeClock{t: time.Unix(1_700_000_000, 0)} + c := newOptInClock(clock.now) + assert.Nil(t, c.totals(), "nothing opted in yet") + + cam1, endCam1 := announce(t, "/.abr/live/cam1") + cam2, endCam2 := announce(t, "/.abr/live/cam2") + hls, _ := announce(t, "/.hls/live/cam1") + plain, _ := announce(t, "/live/cam1") + c.track(cam1) + c.track(plain) + clock.add(10 * time.Second) + c.track(cam2) + c.track(hls) + clock.add(5 * time.Second) + + assert.Equal(t, map[token.OptIn]time.Duration{ + token.OptInABR: 20 * time.Second, // cam1 15 s, cam2 5 s + token.OptInHLS: 5 * time.Second, + }, c.totals(), "live announcements count up to now; a broadcast's own announcement is not an opt-in") + + endCam1() + clock.add(10 * time.Second) + endCam2() + clock.add(time.Hour) + + assert.Equal(t, map[token.OptIn]time.Duration{ + token.OptInABR: 30 * time.Second, // cam1 0–15 s, cam2 10–25 s + token.OptInHLS: time.Hour + 15*time.Second, + }, c.totals(), "an ended announcement stops counting") +} + +func TestOptInClock_EndedAnnouncement(t *testing.T) { + clock := &fakeClock{t: time.Unix(1_700_000_000, 0)} + c := newOptInClock(clock.now) + ann, end := announce(t, "/.abr/live/cam1") + end() + + c.track(ann) + clock.add(time.Minute) + + assert.Nil(t, c.totals(), "an announcement that ended before it was tracked counts nothing") +} + +func TestOptInClock_Nil(t *testing.T) { + var c *optInClock + ann, _ := announce(t, "/.abr/live/cam1") + c.track(ann) + assert.Nil(t, c.totals(), "an unchecked session counts nothing") +} diff --git a/internal/relay/server.go b/internal/relay/server.go index 5aac4784..0181cee8 100644 --- a/internal/relay/server.go +++ b/internal/relay/server.go @@ -275,7 +275,7 @@ func (s *Server) serveUpgrade(w http.ResponseWriter, r *http.Request, upgrader h // so the session counted at connect is closed. if g != nil && !a.served.Load() { s.sessionEnds.add() - s.reportEnd(r.Context(), req, moqt.SessionStats{}, endUpgradeFailed) + s.reportEnd(r.Context(), req, moqt.SessionStats{}, nil, endUpgradeFailed) s.sessionEnds.done() } } @@ -809,7 +809,11 @@ func (s *Server) serveSession(sess *moqt.Session) { a := admissionFrom(sess.Context()) checked := a != nil && a.grant != nil var l *lease + // optIns counts the opt-in time of a checked session, whose usage is + // reported; nil counts none. + var optIns *optInClock if checked { + optIns = newOptInClock(time.Now) a.served.Store(true) // Registered first so that it runs last: after the close below, // with the session's final byte totals and close cause. Counted in @@ -817,7 +821,7 @@ func (s *Server) serveSession(sess *moqt.Session) { s.sessionEnds.add() defer func() { defer s.sessionEnds.done() - s.reportEnd(sess.Context(), a.req, sess.Stats(), sessionEndReason(a, l, context.Cause(sess.Context()))) + s.reportEnd(sess.Context(), a.req, sess.Stats(), optIns, sessionEndReason(a, l, context.Cause(sess.Context()))) }() } // Registered before the close below, so that it runs after it, with the @@ -836,7 +840,7 @@ func (s *Server) serveSession(sess *moqt.Session) { }() defer sess.CloseWithError(moqt.NoError, moqt.NoError.String()) if checked { - if l = startLease(sess.Context(), sess, s.Authorize, a.req, a.grant.Revalidate()); l != nil { + if l = startLease(sess.Context(), sess, s.Authorize, a.req, a.grant.Revalidate(), optIns); l != nil { defer l.stop() } if a.grant.Anonymous() { @@ -874,6 +878,8 @@ func (s *Server) serveSession(sess *moqt.Session) { return } + optIns.track(ann) + slog.Debug("relay: received announcement", "node", s.Config.NodeID, "broadcast_path", ann.BroadcastPath(), diff --git a/usage/usage.go b/usage/usage.go index acc52f60..d434a585 100644 --- a/usage/usage.go +++ b/usage/usage.go @@ -9,6 +9,10 @@ // outnumber publishers by orders of magnitude, so a record per viewer would // make a receiver's load grow with the audience. // +// A publishing session's records also carry how long it kept broadcasts +// opted into each egress (OptInSeconds), so that a control plane can bill an +// option such as ABR by the time it was used. +// // Byte counts are cumulative and from the relay's point of view (what it // received is ingress), so a receiver can take a resent record without // counting it twice. A relay that restarts begins a new Run, whose viewer @@ -46,6 +50,16 @@ const ( EgressBytes = "gateway.egress_bytes" ) +// OptInSeconds names the metric of how long a publishing session kept +// broadcasts opted into the egress named optIn ("hls", "abr"), in whole +// seconds, added up over its broadcasts: two broadcasts opted into ABR for +// a minute each are 120. A session that opted none in carries none. +// +// OptInSeconds("abr") is "gateway.opt_in_seconds.abr" +func OptInSeconds(optIn string) string { + return "gateway.opt_in_seconds." + optIn +} + // Why a publishing session ended, the Reason of its SessionClose. const ( // ReasonRefused is a session the relay ended because a re-check refused diff --git a/usage/usage_test.go b/usage/usage_test.go index 56b41b7b..6f266f6e 100644 --- a/usage/usage_test.go +++ b/usage/usage_test.go @@ -46,3 +46,8 @@ func TestRecord_Validate(t *testing.T) { }) } } + +func TestOptInSeconds(t *testing.T) { + assert.Equal(t, "gateway.opt_in_seconds.abr", OptInSeconds("abr")) + assert.Equal(t, "gateway.opt_in_seconds.hls", OptInSeconds("hls")) +} From 8eb9443b4b6590e78b21bf384ca7007e2042be30 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 11:54:35 +0000 Subject: [PATCH 5/7] feat(ingest): report each pushed stream's usage under its publisher's key With QUMO_USAGE_URL and QUMO_AUTH_KEYS, qumo rtmp and qumo rtsp-push send each stream to the usage URL as a relay sends a publishing session: an open, usage every 30 s and a close, under the kid of the token that granted the push, with the media the ingest took in (ingest.media_bytes) and how long it kept the broadcast opted into each egress. The relay only sees the ingest's own session, so a stream pushed over RTMP or RTSP is billed to its publisher here. - ingest.PublishAuthorizer returns a Permission: the lapse, and the kid of the key that granted it. - Session counts the media pushed into it and its opt-in time (Usage). - The opt-in clock moved from the relay to internal/optin (optin.Clock), which both use. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KPWMJL7PH3rPhK5UTGavaB --- CHANGELOG.md | 1 + docs/site/content/en/docs/cli/rtmp.md | 14 +- docs/site/content/en/docs/cli/rtsp-push.md | 14 +- docs/site/content/en/docs/configuration.md | 2 +- internal/ingest/auth.go | 30 +++-- internal/ingest/auth_test.go | 18 ++- internal/ingest/cmd.go | 84 +++++++++--- .../ingest/fake_publish_authorizer_test.go | 12 +- internal/ingest/rtmp.go | 17 ++- internal/ingest/rtmp_server_test.go | 15 ++- internal/ingest/rtsp.go | 19 ++- internal/ingest/rtsp_test.go | 6 +- internal/ingest/session.go | 17 +++ internal/ingest/session_test.go | 20 +++ internal/ingest/usage.go | 60 +++++++++ internal/ingest/usage_test.go | 123 ++++++++++++++++++ internal/integration/abr_auth_test.go | 26 ++-- internal/integration/optin_usage_test.go | 90 +++++++++++++ .../{relay/optin_time.go => optin/clock.go} | 39 +++--- .../clock_test.go} | 36 ++--- internal/relay/admit.go | 5 +- internal/relay/lease.go | 7 +- internal/relay/server.go | 7 +- usage/usage.go | 15 ++- 24 files changed, 563 insertions(+), 114 deletions(-) create mode 100644 internal/ingest/usage.go create mode 100644 internal/ingest/usage_test.go rename internal/{relay/optin_time.go => optin/clock.go} (54%) rename internal/{relay/optin_time_test.go => optin/clock_test.go} (71%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 430cb6d1..2f99cfdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **`PEERS` names are resolved again while the relay runs, so peers behind a name join and leave without a restart; an entry without a port is an SRV name (`_moqt._udp.hubs.example.com`) (`internal/relay`).** A `PEERS` entry that is neither `host:port` nor a name without a port now stops the relay at startup, and a host that doesn't resolve is retried instead of being dialed by name. - **The relay re-reads its certificates while it runs (`internal/relay`).** About once a minute it reads `CERT_FILE`/`KEY_FILE`, `PEER_CERT_FILE`/`PEER_KEY_FILE` and `CA_FILE` and compares them with what it holds; what changed is taken from the next TLS handshake on, so cert-manager, a Vault or step-ca agent, or any job that writes the files needs no restart. A public certificate is taken once it parses and matches its key; a peer certificate and the CA together, once they pass the startup checks (issued by the CA, the peering name, valid now). Anything else is logged, once per distinct error rather than at every read, and the certificates in use are kept; renewal tools should replace the files atomically (write, then rename). Sessions already established are untouched. The CA is now checked at each handshake on both sides: as the dialed relay (`GetConfigForClient`) and as the dialer (`VerifyConnection` against the current CA, with the same checks crypto/tls made). The relay's key-set and usage requests are signed with the renewed peer certificate (`reqsign.Signer.Rotate`), and `qumo dns` re-reads its `PEER_CERT_FILE`/`PEER_KEY_FILE` on the same one-minute cadence and signs its feed reads with a renewed pair (`internal/certreload`, shared by both). New metrics: `qumo_relay_cert_reloads_total{cert,result}` and `qumo_relay_cert_not_after_seconds{cert}`, and `qumo_dns_cert_reloads_total{cert,result}` and `qumo_dns_cert_not_after_seconds{cert}`; alert on the `not_after` gauges. A CA is rotated by putting the new one beside the old in `CA_FILE` first ([TLS & mTLS](docs/site/content/en/docs/deployment/tls.md#certificate-renewal)). - **A relay reports how long each publishing session kept broadcasts opted into HLS and ABR (`internal/relay`, `usage.OptInSeconds`).** A publishing session's `usage` and `session_close` records now carry `gateway.opt_in_seconds.abr` and `gateway.opt_in_seconds.hls`: the whole seconds its announcements beneath `/.abr/` and `/.hls/` lasted, added up over its broadcasts, from when the relay received each until it ended, the session closed or a revoked ticket ended it. Only an opt-in the session's grant covers reaches the relay, so the time is billed to the key that holds the opt-in, and a control plane can charge for ABR or HLS by the time it was used. `qumo abr` itself never needs to know whose broadcast it transcodes. A session that opted nothing into an egress carries no metric for it, so a receiver that doesn't bill opt-ins sees records as before. +- **RTMP and RTSP ingest report each pushed stream's usage under its publisher's key (`internal/ingest`, `QUMO_USAGE_URL`).** With `QUMO_USAGE_URL` and `QUMO_AUTH_KEYS`, `qumo rtmp` and `qumo rtsp-push` send each stream to the usage URL as a relay sends a publishing session (`internal/meter`): a `session_open`, a `usage` record every 30 s and a `session_close`, under the kid of the token that granted the push, carrying the media the ingest took in (`usage.IngestMediaBytes`, `ingest.media_bytes`: the payload of every frame) and how long it kept the broadcast opted into each egress (`gateway.opt_in_seconds.*`). The relay sees only the ingest's own session, under the ingest's key, so a stream pushed over RTMP or RTSP is billed to its publisher here, as one published over MoQ is billed on the relay. `ingest.PublishAuthorizer` now returns a `Permission`, the lapse and the kid of the key that granted it. The opt-in clock moved from the relay to `internal/optin` (`optin.Clock`), which both use. ### Changed diff --git a/docs/site/content/en/docs/cli/rtmp.md b/docs/site/content/en/docs/cli/rtmp.md index 34fbc5f6..4ac41384 100644 --- a/docs/site/content/en/docs/cli/rtmp.md +++ b/docs/site/content/en/docs/cli/rtmp.md @@ -90,6 +90,17 @@ listener checks no subscriber, so with auth on, viewers go through the relay. $ RELAY_URL=moqt://relay:4433 RELAY_SIGNING_KEY=ingest-key.jwk RELAY_PUBLISH=/live/ qumo rtmp ``` +## Usage + +With `QUMO_USAGE_URL` (and `QUMO_AUTH_KEYS`), the server reports each pushed +stream as a relay reports a publishing session ([usage reports]({{< relref "../configuration#usage-reports" >}})), +under the key of the token that granted the push: a `session_open`, a `usage` +record every 30 s and a `session_close`, carrying the media it took in +(`ingest.media_bytes`, the payload of every frame) and how long it kept its +broadcast opted into each egress (`gateway.opt_in_seconds.abr`, +`gateway.opt_in_seconds.hls`). The relay sees the server's own session, under +the server's key, so the stream is billed here, to its publisher. + ## Configuration | Variable | Default | Description | @@ -97,7 +108,8 @@ $ RELAY_URL=moqt://relay:4433 RELAY_SIGNING_KEY=ingest-key.jwk RELAY_PUBLISH=/li | `RTMP_INGEST_ADDR` | `:1935` | RTMP listen address. | | `RTMP_SERVE_ADDR` | `:4433`; none with `RELAY_URL` | MoQT listen address for subscribers to dial directly. | | `QUMO_AUTH_KEYS` | (unset) | The key set a publisher's credential is verified against, as the relay reads it (a file or URL). Unset accepts every publish. | -| `QUMO_AUTH_KEYS_CACHE` / `QUMO_RELAY_TOKEN` | (unset) | A file a downloaded key set is kept in, and the bearer token sent to a key-set URL. | +| `QUMO_AUTH_KEYS_CACHE` / `QUMO_RELAY_TOKEN` | (unset) | A file a downloaded key set is kept in, and the bearer token sent to a key-set URL and to `QUMO_USAGE_URL`. | +| `QUMO_USAGE_URL` | (unset) | Where each stream's usage is reported, under the key of the publisher's token ([usage](#usage)). Needs `QUMO_AUTH_KEYS`. | | `INGEST_OPT_IN` | (unset) | The egresses (`hls`, `abr`) a stream is opted into on its publisher's behalf, comma-separated: with auth on, those a second `?jwt=` may publish the opt-in of. | | `RELAY_URL` | (unset) | The relay to publish through, dialed as a client (`https://…` or `moqt://…`); it may carry a credential as `?jwt=`. | | `RELAY_SIGNING_KEY`, `RELAY_PUBLISH` | (unset) | Set together: a signing key (`qumo auth keygen`) the server signs a fresh relay credential with on every dial and renews on the session, granting publishing beneath the directory `RELAY_PUBLISH`. A push outside it is refused. | diff --git a/docs/site/content/en/docs/cli/rtsp-push.md b/docs/site/content/en/docs/cli/rtsp-push.md index 0e626e10..d7d9a0f9 100644 --- a/docs/site/content/en/docs/cli/rtsp-push.md +++ b/docs/site/content/en/docs/cli/rtsp-push.md @@ -89,6 +89,17 @@ listener checks no subscriber, so with auth on, viewers go through the relay. $ RELAY_URL=moqt://relay:4433 RELAY_SIGNING_KEY=ingest-key.jwk RELAY_PUBLISH=/live/ qumo rtsp-push ``` +## Usage + +With `QUMO_USAGE_URL` (and `QUMO_AUTH_KEYS`), the server reports each pushed +stream as a relay reports a publishing session ([usage reports]({{< relref "../configuration#usage-reports" >}})), +under the key of the token that granted the push: a `session_open`, a `usage` +record every 30 s and a `session_close`, carrying the media it took in +(`ingest.media_bytes`, the payload of every frame) and how long it kept its +broadcast opted into each egress (`gateway.opt_in_seconds.abr`, +`gateway.opt_in_seconds.hls`). The relay sees the server's own session, under +the server's key, so the stream is billed here, to its publisher. + ## Configuration | Variable | Default | Description | @@ -96,7 +107,8 @@ $ RELAY_URL=moqt://relay:4433 RELAY_SIGNING_KEY=ingest-key.jwk RELAY_PUBLISH=/li | `RTSP_INGEST_ADDR` | `:8554` | RTSP listen address. | | `RTSP_SERVE_ADDR` | `:4433`; none with `RELAY_URL` | MoQT listen address for subscribers to dial directly. | | `QUMO_AUTH_KEYS` | (unset) | The key set a publisher's credential is verified against, as the relay reads it (a file or URL). Unset accepts every publish. | -| `QUMO_AUTH_KEYS_CACHE` / `QUMO_RELAY_TOKEN` | (unset) | A file a downloaded key set is kept in, and the bearer token sent to a key-set URL. | +| `QUMO_AUTH_KEYS_CACHE` / `QUMO_RELAY_TOKEN` | (unset) | A file a downloaded key set is kept in, and the bearer token sent to a key-set URL and to `QUMO_USAGE_URL`. | +| `QUMO_USAGE_URL` | (unset) | Where each stream's usage is reported, under the key of the publisher's token ([usage](#usage)). Needs `QUMO_AUTH_KEYS`. | | `INGEST_OPT_IN` | (unset) | The egresses (`hls`, `abr`) a stream is opted into on its publisher's behalf, comma-separated: with auth on, those a second `?jwt=` may publish the opt-in of. | | `RELAY_URL` | (unset) | The relay to publish through, dialed as a client (`https://…` or `moqt://…`); it may carry a credential as `?jwt=`. | | `RELAY_SIGNING_KEY`, `RELAY_PUBLISH` | (unset) | Set together: a signing key (`qumo auth keygen`) the server signs a fresh relay credential with on every dial and renews on the session, granting publishing beneath the directory `RELAY_PUBLISH`. A push outside it is refused. | diff --git a/docs/site/content/en/docs/configuration.md b/docs/site/content/en/docs/configuration.md index 5a8dfed3..f1a5dc7e 100644 --- a/docs/site/content/en/docs/configuration.md +++ b/docs/site/content/en/docs/configuration.md @@ -265,7 +265,7 @@ Byte counts are cumulative and from the relay's point of view (what it received **Sessions that publish a broadcast are reported one by one,** as the first three records: an open, usage every 30 s, and a close. A session publishes when its token has a `publish` pattern, or from when it is granted a ticket that has one: its open comes then, and what it moved as a viewer up to its last 30 s check stays in its key's viewer total, the rest counting as its usage. -**A publishing session's records also carry its opt-in time:** how long it kept broadcasts opted into each egress, `gateway.opt_in_seconds.abr` for [`qumo abr`](../cli/abr/) and `gateway.opt_in_seconds.hls` for [`qumo hls`](../cli/hls/), in whole seconds, added up over its broadcasts (two broadcasts opted into ABR for a minute each are 120). It is the time each announcement of the session beneath `/.abr/` or `/.hls/` lasted on this relay, from when the relay received it until it ended, the session closed or a revoked ticket ended it, so a control plane can bill an opt-in by the time it was used. Only an opt-in the session's grant covers reaches the relay, so an opt-in a key doesn't hold is never counted. An egress the session opted nothing into has no metric. A broadcast opted in through [RTMP or RTSP ingest](../cli/rtmp/#opting-streams-into-hls-and-abr) counts on the ingest's session, not the publisher's. +**A publishing session's records also carry its opt-in time:** how long it kept broadcasts opted into each egress, `gateway.opt_in_seconds.abr` for [`qumo abr`](../cli/abr/) and `gateway.opt_in_seconds.hls` for [`qumo hls`](../cli/hls/), in whole seconds, added up over its broadcasts (two broadcasts opted into ABR for a minute each are 120). It is the time each announcement of the session beneath `/.abr/` or `/.hls/` lasted on this relay, from when the relay received it until it ended, the session closed or a revoked ticket ended it, so a control plane can bill an opt-in by the time it was used. Only an opt-in the session's grant covers reaches the relay, so an opt-in a key doesn't hold is never counted. An egress the session opted nothing into has no metric. A broadcast opted in through [RTMP or RTSP ingest](../cli/rtmp/#opting-streams-into-hls-and-abr) counts on the ingest's session, under the ingest's own key; the ingest reports the stream itself under its publisher's key ([rtmp → usage](../cli/rtmp/#usage)), so a control plane bills the publisher's key and leaves the ingest's out, as it does any key of its own. **Every other session is a viewer, and viewers are reported together.** Viewers outnumber publishers by orders of magnitude, and a record per viewer would make the receiver's load grow with the audience. Their bytes are added up per key into one running total since the relay started, sent as a `viewers` record naming the key and the relay's `run`. The `run` is new each time the relay starts, so a restart begins new totals rather than lowering old ones. There is no open or close for viewers, and a total is sent only when it has changed. diff --git a/internal/ingest/auth.go b/internal/ingest/auth.go index 4e9ab881..db01e205 100644 --- a/internal/ingest/auth.go +++ b/internal/ingest/auth.go @@ -17,12 +17,22 @@ import ( // PublishAuthorizer decides whether a pushed stream may publish the broadcast // at path with credential, a ?jwt= the publisher presented ("" for none), -// and returns when that permission lapses: the stream is cut then. The zero -// time never lapses. An error refuses the stream. +// and returns the permission it grants. An error refuses the stream. // // A stream's opt-ins (see [optIn]) are asked for as well, with path the // opt-in's, such as /.abr/live/cam for /live/cam. -type PublishAuthorizer func(credential string, path moqt.BroadcastPath) (lapses time.Time, err error) +type PublishAuthorizer func(credential string, path moqt.BroadcastPath) (Permission, error) + +// A Permission is what a PublishAuthorizer grants a push. +type Permission struct { + // Lapses is when the permission lapses: the stream is cut then. The zero + // time never lapses. + Lapses time.Time + // KeyID is the kid of the key that signed the credential granting it, + // whose usage the push is (see [RTMPConfig.Meter]), or "" when none was + // checked. + KeyID string +} // splitCredential splits a name such as "cam?jwt=A&jwt=B" into the name and // the credentials its query carries as jwt, in order, none for none. @@ -65,21 +75,21 @@ func checkPushPath(path moqt.BroadcastPath) error { // credentials, in order, and returns the first permission granted. Without // credentials it asks for a publish that presented none. When none is // granted it returns the first refusal. -func authorizeAny(authorize PublishAuthorizer, credentials []string, path moqt.BroadcastPath) (time.Time, error) { +func authorizeAny(authorize PublishAuthorizer, credentials []string, path moqt.BroadcastPath) (Permission, error) { if len(credentials) == 0 { return authorize("", path) } var first error for _, credential := range credentials { - lapses, err := authorize(credential, path) + p, err := authorize(credential, path) if err == nil { - return lapses, nil + return p, nil } if first == nil { first = err } } - return time.Time{}, first + return Permission{}, first } // optIn opts sess's broadcast into each of optIns, the egresses the server @@ -91,15 +101,15 @@ func authorizeAny(authorize PublishAuthorizer, credentials []string, path moqt.B func optIn(sess *Session, authorize PublishAuthorizer, credentials []string, optIns []token.OptIn) { for _, o := range optIns { path := optin.Path(o.Directory(), sess.path) - var lapses time.Time + var granted Permission if authorize != nil { var err error - if lapses, err = authorizeAny(authorize, credentials, path); err != nil { + if granted, err = authorizeAny(authorize, credentials, path); err != nil { slog.Info("ingest opt-in refused", "broadcast_path", sess.path, "opt_in", o, "error", err) continue } } - sess.OptIn(o, lapses) + sess.OptIn(o, granted.Lapses) } } diff --git a/internal/ingest/auth_test.go b/internal/ingest/auth_test.go index 96759acb..e9792556 100644 --- a/internal/ingest/auth_test.go +++ b/internal/ingest/auth_test.go @@ -82,11 +82,11 @@ func TestAuthorizeAny(t *testing.T) { } for name, tt := range tests { t.Run(name, func(t *testing.T) { - fake := &fakePublishAuthorizer{answer: func(ask publishAsk) (time.Time, error) { + fake := &fakePublishAuthorizer{answer: func(ask publishAsk) (Permission, error) { if err := tt.answers[ask.credential]; err != nil { - return time.Time{}, err + return Permission{}, err } - return lapses, nil + return Permission{Lapses: lapses, KeyID: ask.credential}, nil }} got, err := authorizeAny(fake.authorize, tt.credentials, "/live/cam") @@ -101,7 +101,8 @@ func TestAuthorizeAny(t *testing.T) { return } require.NoError(t, err) - assert.Equal(t, lapses, got) + assert.Equal(t, Permission{Lapses: lapses, KeyID: tt.wantAsked[len(tt.wantAsked)-1]}, got, + "the permission of the credential that granted it") }) } } @@ -169,7 +170,8 @@ func TestPublishAuthorizer(t *testing.T) { authorize := publishAuthorizer(tt.verifier, tt.beneath) require.NotNil(t, authorize) - lapses, err := authorize(tt.credential, tt.path) + granted, err := authorize(tt.credential, tt.path) + lapses := granted.Lapses switch { case tt.wantErr != nil: @@ -181,6 +183,7 @@ func TestPublishAuthorizer(t *testing.T) { case tt.wantLapse: require.NoError(t, err) assert.WithinDuration(t, time.Now().Add(time.Minute), lapses, 5*time.Second, "cut when the credential expires") + assert.Equal(t, tt.verifier == v, granted.KeyID == key.ID, "the push is the usage of the key that signed it") case tt.verifier == nil: require.NoError(t, err) assert.True(t, lapses.IsZero(), "auth off never cuts") @@ -219,9 +222,10 @@ func TestPublishAuthorizer_OptIn(t *testing.T) { } t.Run("a key that holds the opt-in", func(t *testing.T) { - lapses, err := publishAuthorizer(paid, "/live/")(sign(paidKey), "/.abr/live/cam") + granted, err := publishAuthorizer(paid, "/live/")(sign(paidKey), "/.abr/live/cam") require.NoError(t, err) - assert.WithinDuration(t, time.Now().Add(time.Minute), lapses, 5*time.Second) + assert.WithinDuration(t, time.Now().Add(time.Minute), granted.Lapses, 5*time.Second) + assert.Equal(t, paidKey.ID, granted.KeyID) }) t.Run("a key that does not", func(t *testing.T) { _, err := publishAuthorizer(unpaid, "/live/")(sign(unpaidKey), "/.abr/live/cam") diff --git a/internal/ingest/cmd.go b/internal/ingest/cmd.go index 87583c9f..8372ea53 100644 --- a/internal/ingest/cmd.go +++ b/internal/ingest/cmd.go @@ -19,6 +19,7 @@ import ( "github.com/qumo-dev/qumo/internal/auth" "github.com/qumo-dev/qumo/internal/cors" "github.com/qumo-dev/qumo/internal/envconfig" + "github.com/qumo-dev/qumo/internal/meter" "github.com/qumo-dev/qumo/internal/optin" "github.com/qumo-dev/qumo/internal/quicconfig" "github.com/qumo-dev/qumo/internal/upstream" @@ -41,7 +42,8 @@ const ( // RTMP_SERVE_ADDR - MoQT listen address for subscribers to dial directly (default: ":4433" without RELAY_URL, none with it) // QUMO_AUTH_KEYS - the key set a publisher's credential (?jwt= on the stream key) is verified against, as the relay reads it (default: none, every publish is accepted) // QUMO_AUTH_KEYS_CACHE - a file a downloaded key set is kept in between runs -// QUMO_RELAY_TOKEN - bearer token sent to a key-set URL +// QUMO_RELAY_TOKEN - bearer token sent to a key-set URL and to QUMO_USAGE_URL +// QUMO_USAGE_URL - where each stream's usage is reported, under the key of the publisher's token: the media it took in and its opt-in time (package usage); needs QUMO_AUTH_KEYS (default: none, no reports) // INGEST_OPT_IN - the egresses (hls, abr) a stream is opted into on its publisher's behalf, comma-separated: with auth on, those a second ?jwt= on the stream key may publish the opt-in of (/.abr/live/cam for /live/cam); with it off, every stream's (default: none) // RELAY_URL, RELAY_SIGNING_KEY, RELAY_PUBLISH, RELAY_CA_FILE, RELAY_TLS_INSECURE // - the relay to publish through (upstream.FromEnv lists them) @@ -51,8 +53,8 @@ const ( // QUIC_* - QUIC transport settings (quicconfig.FromEnv lists them) func RunRTMP(_ []string) error { return runPush("rtmp", "RTMP_INGEST_ADDR", defaultRTMPIngestAddr, "RTMP_SERVE_ADDR", - func(addr string, mux *moqt.TrackMux, authorize PublishAuthorizer, optIns []token.OptIn) pushServer { - return NewRTMPServer(RTMPConfig{Addr: addr, TrackMux: mux, Authorize: authorize, OptIns: optIns}) + func(c pushConfig) pushServer { + return NewRTMPServer(RTMPConfig{Addr: c.addr, TrackMux: c.mux, Authorize: c.authorize, OptIns: c.optIns, Meter: c.meter}) }) } @@ -65,8 +67,8 @@ func RunRTMP(_ []string) error { // on the URL it pushes to. func RunRTSP(_ []string) error { return runPush("rtsp", "RTSP_INGEST_ADDR", defaultRTSPIngestAddr, "RTSP_SERVE_ADDR", - func(addr string, mux *moqt.TrackMux, authorize PublishAuthorizer, optIns []token.OptIn) pushServer { - return NewRTSPServer(RTSPConfig{Addr: addr, TrackMux: mux, Authorize: authorize, OptIns: optIns}) + func(c pushConfig) pushServer { + return NewRTSPServer(RTSPConfig{Addr: c.addr, TrackMux: c.mux, Authorize: c.authorize, OptIns: c.optIns, Meter: c.meter}) }) } @@ -81,10 +83,18 @@ var ( _ pushServer = (*RTSPServer)(nil) ) +// pushConfig is what runPush gives the push server it makes. +type pushConfig struct { + addr string + mux *moqt.TrackMux + authorize PublishAuthorizer + optIns []token.OptIn + meter *meter.Reporter // nil: no usage reports +} + // runPush runs the push ingest server newServer makes, named name, until // SIGINT or SIGTERM, with its broadcasts published as originFromEnv says. -func runPush(name, ingestEnv, defaultIngestAddr, serveEnv string, - newServer func(addr string, mux *moqt.TrackMux, authorize PublishAuthorizer, optIns []token.OptIn) pushServer) error { +func runPush(name, ingestEnv, defaultIngestAddr, serveEnv string, newServer func(pushConfig) pushServer) error { ingestAddr := envconfig.String(ingestEnv, defaultIngestAddr) quicConfig, err := quicconfig.FromEnv(nil) if err != nil { @@ -103,12 +113,35 @@ func runPush(name, ingestEnv, defaultIngestAddr, serveEnv string, if err != nil { return err } + m, err := meterFromEnv(name) + if err != nil { + return err + } + // The meter sends what is left once more when meterCtx ends, after the + // server has shut down and every stream has recorded its close. + meterCtx, stopMeter := context.WithCancel(context.Background()) + meterDone := make(chan struct{}) + go func() { + defer close(meterDone) + if m != nil { + m.Run(meterCtx) + } + }() + defer func() { + stopMeter() + <-meterDone + }() trackMux := moqt.NewTrackMux(0) - srv := newServer(ingestAddr, trackMux, authorize, out.optIns) + srv := newServer(pushConfig{addr: ingestAddr, mux: trackMux, authorize: authorize, optIns: out.optIns, meter: m}) log.Println(" Ingest :", ingestAddr) out.logBanner() log.Println(" Auth :", authName) + usageURL := "none" + if m != nil { + usageURL = upstream.RelayName(os.Getenv("QUMO_USAGE_URL")) + } + log.Println(" Usage :", usageURL) go func() { if err := srv.ListenAndServe(ctx); err != nil && ctx.Err() == nil { @@ -265,6 +298,25 @@ func publishAuthorizerFromEnv(ctx context.Context, name, beneath string) (Publis return publishAuthorizer(v, beneath), v.Source(), nil } +// meterFromEnv returns the meter that reports the usage of the ingest +// command name's streams to QUMO_USAGE_URL, or nil when it is unset. Usage +// is reported under the key of each publisher's token, so it needs a key set +// (QUMO_AUTH_KEYS). +func meterFromEnv(name string) (*meter.Reporter, error) { + usageURL := os.Getenv("QUMO_USAGE_URL") + if usageURL == "" { + return nil, nil + } + if os.Getenv("QUMO_AUTH_KEYS") == "" { + return nil, fmt.Errorf("%s: QUMO_USAGE_URL needs QUMO_AUTH_KEYS: a stream's usage is reported under the key of its publisher's token", name) + } + m, err := meter.New(meter.Config{Name: name, URL: usageURL, Token: os.Getenv("QUMO_RELAY_TOKEN")}) + if err != nil { + return nil, fmt.Errorf("%s: QUMO_USAGE_URL: %w", name, err) + } + return m, nil +} + // errOutsidePublish refuses a push the relay would not take from the ingest // server: its path lies outside RELAY_PUBLISH. var errOutsidePublish = errors.New("the path lies outside RELAY_PUBLISH, beneath which the relay takes this server's broadcasts") @@ -288,33 +340,33 @@ func publishAuthorizer(v *auth.Verifier, beneath string) PublishAuthorizer { if v == nil && beneath == "" { return nil } - return func(credential string, path moqt.BroadcastPath) (time.Time, error) { + return func(credential string, path moqt.BroadcastPath) (Permission, error) { broadcast := path if o, ok := optin.Of(path); ok { b, named := optin.Broadcast(string(path), o.Directory()) if !named { // Cleaned, it could name a broadcast outside the // directory the relay grants its exact path in. - return time.Time{}, fmt.Errorf("%s: %w", path, errOptInNamesNone) + return Permission{}, fmt.Errorf("%s: %w", path, errOptInNamesNone) } broadcast = moqt.BroadcastPath(b) } if beneath != "" && !strings.HasPrefix(string(broadcast), beneath) { - return time.Time{}, fmt.Errorf("%s: %w (%s)", path, errOutsidePublish, beneath) + return Permission{}, fmt.Errorf("%s: %w (%s)", path, errOutsidePublish, beneath) } if v == nil { - return time.Time{}, nil + return Permission{}, nil } if credential == "" { - return time.Time{}, errNoCredential + return Permission{}, errNoCredential } g, err := v.Check(credential, token.ActionPublish, path) if err != nil { - return time.Time{}, err + return Permission{}, err } if !g.Allows(token.ActionPublish, path) { - return time.Time{}, fmt.Errorf("the credential may not publish %s", path) + return Permission{}, fmt.Errorf("the credential may not publish %s", path) } - return g.Expires(), nil + return Permission{Lapses: g.Expires(), KeyID: g.KeyID()}, nil } } diff --git a/internal/ingest/fake_publish_authorizer_test.go b/internal/ingest/fake_publish_authorizer_test.go index 396e8e0f..97ce0d10 100644 --- a/internal/ingest/fake_publish_authorizer_test.go +++ b/internal/ingest/fake_publish_authorizer_test.go @@ -14,13 +14,14 @@ type publishAsk struct { path moqt.BroadcastPath } -// fakePublishAuthorizer answers every publish with lapses and err, or with +// fakePublishAuthorizer answers every publish with lapses and keyID, or err, or with // answer when it is set, and records what it was asked. The servers ask from // their connection goroutines, so the record is guarded. type fakePublishAuthorizer struct { lapses time.Time + keyID string err error - answer func(publishAsk) (time.Time, error) + answer func(publishAsk) (Permission, error) mu sync.Mutex asked []publishAsk @@ -28,7 +29,7 @@ type fakePublishAuthorizer struct { var _ PublishAuthorizer = (*fakePublishAuthorizer)(nil).authorize -func (f *fakePublishAuthorizer) authorize(credential string, path moqt.BroadcastPath) (time.Time, error) { +func (f *fakePublishAuthorizer) authorize(credential string, path moqt.BroadcastPath) (Permission, error) { f.mu.Lock() defer f.mu.Unlock() ask := publishAsk{credential: credential, path: path} @@ -36,7 +37,10 @@ func (f *fakePublishAuthorizer) authorize(credential string, path moqt.Broadcast if f.answer != nil { return f.answer(ask) } - return f.lapses, f.err + if f.err != nil { + return Permission{}, f.err + } + return Permission{Lapses: f.lapses, KeyID: f.keyID}, nil } // questions returns what the fake was asked so far. diff --git a/internal/ingest/rtmp.go b/internal/ingest/rtmp.go index b8f9a1db..d3e8d3c7 100644 --- a/internal/ingest/rtmp.go +++ b/internal/ingest/rtmp.go @@ -6,11 +6,12 @@ import ( "log/slog" "net" "sync" - "time" "github.com/qumo-dev/gomoqt/moqt" + "github.com/qumo-dev/qumo/internal/meter" "github.com/qumo-dev/qumo/internal/rtmp" "github.com/qumo-dev/qumo/token" + "github.com/qumo-dev/qumo/usage" ) // RTMPConfig holds configuration for an [RTMPServer]. @@ -32,6 +33,13 @@ type RTMPConfig struct { // is asked for the opt-in's path, such as /.abr/live/cam): a publisher // presents a second ?jwt= for it. Nil opts none in. OptIns []token.OptIn + + // Meter, when set, reports each stream's usage under the key of the + // credential Authorize granted it with (Permission.KeyID): an open, + // the media it took in and its opt-in time every 30 s, and a close + // (usage.IngestMediaBytes, usage.OptInSeconds). A stream no key granted, + // with auth off, is not reported. + Meter *meter.Reporter } // RTMPServer accepts RTMP publish connections and bridges each stream to @@ -153,7 +161,7 @@ func (s *RTMPServer) handleConn(ctx context.Context, conn *rtmp.Conn) { var ( path moqt.BroadcastPath credentials []string - lapses time.Time + granted Permission ) mr, err := conn.AcceptStreamFunc(func(app, streamKey string) error { path, credentials = rtmpTarget(app, streamKey) @@ -164,7 +172,7 @@ func (s *RTMPServer) handleConn(ctx context.Context, conn *rtmp.Conn) { return nil } var err error - lapses, err = authorizeAny(s.config.Authorize, credentials, path) + granted, err = authorizeAny(s.config.Authorize, credentials, path) return err }) if err != nil { @@ -178,7 +186,7 @@ func (s *RTMPServer) handleConn(ctx context.Context, conn *rtmp.Conn) { } defer mr.Close() - ctx, cancel := untilLapse(ctx, lapses) + ctx, cancel := untilLapse(ctx, granted.Lapses) defer cancel() // A read blocked on the connection doesn't see ctx end; closing the // connection ends it. @@ -198,6 +206,7 @@ func (s *RTMPServer) handleConn(ctx context.Context, conn *rtmp.Conn) { } defer sess.Close() optIn(sess, s.config.Authorize, credentials, s.config.OptIns) + defer meterStream(s.config.Meter, granted.KeyID, sess)(usage.ReasonClosed) slog.Info("RTMP ingest started", "remote", conn.RemoteAddr(), diff --git a/internal/ingest/rtmp_server_test.go b/internal/ingest/rtmp_server_test.go index 398f542f..48675170 100644 --- a/internal/ingest/rtmp_server_test.go +++ b/internal/ingest/rtmp_server_test.go @@ -23,6 +23,13 @@ func startRTMPServer(t *testing.T, authorize PublishAuthorizer, optIns ...token. t.Helper() mux := moqt.NewTrackMux(0) server := NewRTMPServer(RTMPConfig{Addr: "127.0.0.1:0", TrackMux: mux, Authorize: authorize, OptIns: optIns}) + return mux, serveRTMP(t, server) +} + +// serveRTMP serves server until test cleanup, and returns its address once +// it is listening. +func serveRTMP(t *testing.T, server *RTMPServer) net.Addr { + t.Helper() ctx, cancel := context.WithCancel(context.Background()) errCh := make(chan error, 1) go func() { errCh <- server.ListenAndServe(ctx) }() @@ -36,7 +43,7 @@ func startRTMPServer(t *testing.T, authorize PublishAuthorizer, optIns ...token. addr = server.Addr() return addr != nil }, 2*time.Second, 10*time.Millisecond, "RTMP server never started listening") - return mux, addr + return addr } // publishRTMP opens a publish of app/streamKey on the server at addr. The @@ -121,11 +128,11 @@ func TestRTMPServer_OptIns(t *testing.T) { t.Run(name, func(t *testing.T) { var authorize PublishAuthorizer if tt.authorize { - fake := &fakePublishAuthorizer{answer: func(ask publishAsk) (time.Time, error) { + fake := &fakePublishAuthorizer{answer: func(ask publishAsk) (Permission, error) { if grants[ask] { - return time.Time{}, nil + return Permission{}, nil } - return time.Time{}, errNoGrant + return Permission{}, errNoGrant }} authorize = fake.authorize } diff --git a/internal/ingest/rtsp.go b/internal/ingest/rtsp.go index 2cdeab21..9278b482 100644 --- a/internal/ingest/rtsp.go +++ b/internal/ingest/rtsp.go @@ -13,8 +13,10 @@ import ( "time" "github.com/qumo-dev/gomoqt/moqt" + "github.com/qumo-dev/qumo/internal/meter" "github.com/qumo-dev/qumo/internal/rtsp" "github.com/qumo-dev/qumo/token" + "github.com/qumo-dev/qumo/usage" ) // RTSPConfig holds configuration for an [RTSPServer]. @@ -34,6 +36,10 @@ type RTSPConfig struct { // OptIns are the egresses each stream is opted into, as // [RTMPConfig.OptIns] are. OptIns []token.OptIn + + // Meter, when set, reports each stream's usage under the key that + // granted it, as [RTMPConfig.Meter] does. + Meter *meter.Reporter } // RTSPServer accepts RTSP push connections and bridges each stream to @@ -130,10 +136,13 @@ func (s *RTSPServer) handleConn(ctx context.Context, conn *rtsp.Conn) { sess *Session tracks = make(map[uint8]*rtspTrack) // channel -> track sdp *rtsp.SDP + // endUsage ends the usage report of sess. + endUsage = func(string) {} ) defer func() { if sess != nil { + endUsage(usage.ReasonClosed) sess.Close() } }() @@ -198,8 +207,9 @@ func (s *RTSPServer) handleConn(ctx context.Context, conn *rtsp.Conn) { break } credentials := req.URL.Query()["jwt"] + var granted Permission if s.config.Authorize != nil { - lapses, err := authorizeAny(s.config.Authorize, credentials, path) + granted, err = authorizeAny(s.config.Authorize, credentials, path) if err != nil { slog.Warn("RTSP publish refused", "remote", conn.RemoteAddr(), @@ -213,8 +223,8 @@ func (s *RTSPServer) handleConn(ctx context.Context, conn *rtsp.Conn) { cut.Stop() cut = nil } - if !lapses.IsZero() { - cut = time.AfterFunc(time.Until(lapses), func() { + if !granted.Lapses.IsZero() { + cut = time.AfterFunc(time.Until(granted.Lapses), func() { slog.Info("RTSP ingest cut: the publisher's credential lapsed", "remote", conn.RemoteAddr(), "broadcast_path", path, @@ -228,15 +238,18 @@ func (s *RTSPServer) handleConn(ctx context.Context, conn *rtsp.Conn) { // one leaves the connection as it was. sdp = rtsp.ParseSDP(string(body)) if sess != nil { + endUsage(usage.ReasonClosed) sess.Close() } sess, err = NewSession(s.config.TrackMux, path) if err != nil { slog.Error("failed to create ingest session", "error", err) resp.StatusCode = rtsp.StatusInternalServerError + endUsage = func(string) {} break } optIn(sess, s.config.Authorize, credentials, s.config.OptIns) + endUsage = meterStream(s.config.Meter, granted.KeyID, sess) case rtsp.MethodSetup: if sdp == nil || sess == nil { diff --git a/internal/ingest/rtsp_test.go b/internal/ingest/rtsp_test.go index da1f4499..9ce54e42 100644 --- a/internal/ingest/rtsp_test.go +++ b/internal/ingest/rtsp_test.go @@ -241,11 +241,11 @@ func TestRTSPServer_Authorize(t *testing.T) { // second ?jwt= on its URL that may publish the opt-in, asked for every // credential in order. func TestRTSPServer_OptIns(t *testing.T) { - authorize := &fakePublishAuthorizer{answer: func(ask publishAsk) (time.Time, error) { + authorize := &fakePublishAuthorizer{answer: func(ask publishAsk) (Permission, error) { if ask.path == "/.abr/live/cam" && ask.credential != "abr" { - return time.Time{}, errors.New("no grant") + return Permission{}, errors.New("no grant") } - return time.Time{}, nil + return Permission{}, nil }} server, addr := startRTSPServer(t, authorize.authorize, token.OptInABR) rc := dialRTSP(t, addr) diff --git a/internal/ingest/session.go b/internal/ingest/session.go index 670724cc..b742a3d0 100644 --- a/internal/ingest/session.go +++ b/internal/ingest/session.go @@ -25,6 +25,7 @@ import ( "context" "fmt" "log/slog" + "sync/atomic" "time" "github.com/qumo-dev/gomoqt/moqt" @@ -48,6 +49,10 @@ type Session struct { // ctx ends when the session closes, and with it every opt-in. ctx context.Context cancel context.CancelFunc + // mediaBytes counts the media pushed: the payload of every frame. + mediaBytes atomic.Uint64 + // optIns counts how long the session's opt-ins last. + optIns *optin.Clock } // NewSession creates a new ingest session that announces path on trackMux. @@ -79,6 +84,7 @@ func NewSession(trackMux *moqt.TrackMux, path moqt.BroadcastPath) (*Session, err endAnn: endAnn, ctx: ctx, cancel: cancel, + optIns: optin.NewClock(time.Now), }, nil } @@ -100,9 +106,17 @@ func (s *Session) OptIn(o token.OptIn, lapses time.Time) { slog.Info("ingest opt-in withdrawn", "broadcast_path", s.path, "opt_in", o) }) s.trackMux.Announce(ann, optin.Handler(s.path)) + s.optIns.Track(ann) slog.Info("ingest opted in", "broadcast_path", s.path, "opt_in", o, "opt_in_path", path) } +// Usage returns what the session has taken in so far: the bytes of media +// pushed into it, and how long it has kept its broadcast opted into each +// egress (nil for none). +func (s *Session) Usage() (mediaBytes uint64, optIns map[token.OptIn]time.Duration) { + return s.mediaBytes.Load(), s.optIns.Totals() +} + // RegisterVideo registers (or replaces) the video track in the MSF catalog // using codec configuration extracted from a sequence header. Subscribers // can discover the track via the catalog and receive video frames pushed @@ -125,6 +139,7 @@ func (s *Session) RegisterAudio(cfg *AACConfig) error { // // timestampUS is the presentation timestamp in microseconds. func (s *Session) PushVideo(timestampUS int64, data []byte, isKeyframe bool) { + s.mediaBytes.Add(uint64(len(data))) f := moqt.NewFrame(mediaFrameSize(timestampUS, len(data))) writeMediaFrame(f, timestampUS, data) s.handler.video.push(f, isKeyframe, timestampUS) @@ -135,6 +150,7 @@ func (s *Session) PushVideo(timestampUS int64, data []byte, isKeyframe bool) { // // timestampUS is the presentation timestamp in microseconds. func (s *Session) PushAudio(timestampUS int64, data []byte) { + s.mediaBytes.Add(uint64(len(data))) f := moqt.NewFrame(mediaFrameSize(timestampUS, len(data))) writeMediaFrame(f, timestampUS, data) s.handler.audio.push(f) @@ -156,6 +172,7 @@ func (s *Session) PushAudioFrames(timestampsUS []int64, frames [][]byte) { } built := make([]*moqt.Frame, len(frames)) for i, data := range frames { + s.mediaBytes.Add(uint64(len(data))) pts := timestampsUS[i] f := moqt.NewFrame(mediaFrameSize(pts, len(data))) writeMediaFrame(f, pts, data) diff --git a/internal/ingest/session_test.go b/internal/ingest/session_test.go index ba4b3a3b..13791c0a 100644 --- a/internal/ingest/session_test.go +++ b/internal/ingest/session_test.go @@ -57,3 +57,23 @@ func TestSession_OptIn(t *testing.T) { assert.Eventually(t, func() bool { return !published(mux, "/.hls/live/cam") }, 3*time.Second, 20*time.Millisecond, "withdrawn with the session") } + +// TestSession_Usage verifies a session counts the media pushed into it, the +// payload of every frame, and how long its opt-ins last. +func TestSession_Usage(t *testing.T) { + sess, err := NewSession(moqt.NewTrackMux(0), "/live/cam") + require.NoError(t, err) + defer sess.Close() + + sess.PushVideo(0, make([]byte, 1000), true) + sess.PushAudio(0, make([]byte, 100)) + sess.PushAudioFrames([]int64{0, 21}, [][]byte{make([]byte, 10), make([]byte, 20)}) + media, optIns := sess.Usage() + assert.Equal(t, uint64(1130), media) + assert.Nil(t, optIns, "opted into nothing") + + sess.OptIn(token.OptInABR, time.Time{}) + time.Sleep(20 * time.Millisecond) + _, optIns = sess.Usage() + assert.Greater(t, optIns[token.OptInABR], time.Duration(0), "an opt-in counts while it lasts") +} diff --git a/internal/ingest/usage.go b/internal/ingest/usage.go new file mode 100644 index 00000000..adc9e498 --- /dev/null +++ b/internal/ingest/usage.go @@ -0,0 +1,60 @@ +package ingest + +import ( + "context" + "crypto/rand" + "math" + "time" + + "github.com/qumo-dev/qumo/internal/meter" + "github.com/qumo-dev/qumo/usage" +) + +// usageInterval is how often a pushed stream's usage is reported while it +// runs, as a relay revalidates a session. +const usageInterval = 30 * time.Second + +// meterStream reports the usage of sess, a stream pushed under the key kid, +// to m until the returned function is called with why it ended: an open now, +// its cumulative usage every usageInterval, and a close with its final +// totals. With no meter or no key (auth off) it reports nothing. +func meterStream(m *meter.Reporter, kid string, sess *Session) (end func(reason string)) { + if m == nil || kid == "" { + return func(string) {} + } + id := rand.Text() + m.Open(id, kid) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + ticker := time.NewTicker(usageInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + m.Report(id, kid, streamMetrics(sess)) + } + } + }() + return func(reason string) { + cancel() + <-done + m.Close(id, kid, streamMetrics(sess), reason) + } +} + +// streamMetrics is a pushed stream's usage: the media it took in, and how +// long it kept its broadcast opted into each egress, in whole seconds. +func streamMetrics(sess *Session) map[string]int64 { + mediaBytes, optIns := sess.Usage() + m := map[string]int64{usage.IngestMediaBytes: int64(min(mediaBytes, math.MaxInt64))} + for o, d := range optIns { + if secs := int64(d / time.Second); secs > 0 { + m[usage.OptInSeconds(string(o))] = secs + } + } + return m +} diff --git a/internal/ingest/usage_test.go b/internal/ingest/usage_test.go new file mode 100644 index 00000000..249c5378 --- /dev/null +++ b/internal/ingest/usage_test.go @@ -0,0 +1,123 @@ +package ingest + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/qumo-dev/gomoqt/moqt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/qumo-dev/qumo/internal/meter" + "github.com/qumo-dev/qumo/token" + "github.com/qumo-dev/qumo/usage" +) + +// usageSink records the usage records POSTed to it. +type usageSink struct { + mu sync.Mutex + records []usage.Record +} + +func (s *usageSink) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var batch []usage.Record + if err := json.NewDecoder(r.Body).Decode(&batch); err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.records = append(s.records, batch...) +} + +func (s *usageSink) all() []usage.Record { + s.mu.Lock() + defer s.mu.Unlock() + return append([]usage.Record(nil), s.records...) +} + +// newTestMeter returns a meter sending to a fresh sink. +func newTestMeter(t *testing.T) (*meter.Reporter, *usageSink) { + t.Helper() + sink := &usageSink{} + srv := httptest.NewServer(sink) + t.Cleanup(srv.Close) + m, err := meter.New(meter.Config{Name: "test", URL: srv.URL}) + require.NoError(t, err) + return m, sink +} + +// TestMeterStream verifies a pushed stream is reported under its key: an +// open, and a close carrying the media it took in and its opt-in time. +func TestMeterStream(t *testing.T) { + m, sink := newTestMeter(t) + sess, err := NewSession(moqt.NewTrackMux(0), "/live/cam") + require.NoError(t, err) + defer sess.Close() + + end := meterStream(m, "k1", sess) + sess.PushVideo(0, make([]byte, 500), true) + sess.OptIn(token.OptInABR, time.Time{}) + time.Sleep(1100 * time.Millisecond) + end(usage.ReasonClosed) + require.NoError(t, m.Flush(t.Context())) + + records := sink.all() + require.Len(t, records, 2) + assert.Equal(t, usage.SessionOpen, records[0].Type) + closed := records[1] + assert.Equal(t, usage.SessionClose, closed.Type) + assert.Equal(t, records[0].SessionID, closed.SessionID) + assert.Equal(t, "k1", closed.KID) + assert.Equal(t, usage.ReasonClosed, closed.Reason) + assert.Equal(t, int64(500), closed.Metrics[usage.IngestMediaBytes]) + assert.Equal(t, int64(1), closed.Metrics[usage.OptInSeconds("abr")]) + assert.NotContains(t, closed.Metrics, usage.OptInSeconds("hls")) + for _, rec := range records { + assert.NoError(t, rec.Validate()) + } +} + +// TestMeterStream_Unreported verifies a stream with no meter, or no key +// (auth off), is not reported. +func TestMeterStream_Unreported(t *testing.T) { + m, sink := newTestMeter(t) + sess, err := NewSession(moqt.NewTrackMux(0), "/live/cam") + require.NoError(t, err) + defer sess.Close() + + meterStream(nil, "k1", sess)(usage.ReasonClosed) + meterStream(m, "", sess)(usage.ReasonClosed) + require.NoError(t, m.Flush(t.Context())) + + assert.Empty(t, sink.all()) +} + +// TestRTMPServer_Meter verifies the RTMP server reports a stream under the +// key that granted it, and a refused one not at all. +func TestRTMPServer_Meter(t *testing.T) { + m, sink := newTestMeter(t) + authorize := &fakePublishAuthorizer{answer: func(ask publishAsk) (Permission, error) { + if ask.credential != "tok" { + return Permission{}, errNoCredential + } + return Permission{KeyID: "tenant-key"}, nil + }} + mux := moqt.NewTrackMux(0) + server := NewRTMPServer(RTMPConfig{Addr: "127.0.0.1:0", TrackMux: mux, Authorize: authorize.authorize, Meter: m}) + addr := serveRTMP(t, server) + + publishRTMP(t, addr, "live", "refused?jwt=nope") + publishRTMP(t, addr, "live", "cam?jwt=tok") + require.Eventually(t, func() bool { return published(mux, "/live/cam") }, 3*time.Second, 10*time.Millisecond) + require.NoError(t, m.Flush(t.Context())) + + records := sink.all() + require.Len(t, records, 1, "an open for the stream granted; none for the refused one") + assert.Equal(t, usage.SessionOpen, records[0].Type) + assert.Equal(t, "tenant-key", records[0].KID) +} diff --git a/internal/integration/abr_auth_test.go b/internal/integration/abr_auth_test.go index 7f96ed81..06b94a34 100644 --- a/internal/integration/abr_auth_test.go +++ b/internal/integration/abr_auth_test.go @@ -139,6 +139,21 @@ func runPaidOptIn(t *testing.T, optIns []token.OptIn, paid bool) { "the transcoder, on its own credential, did not publish the ABR broadcast") } +// verifyingAuthorizer checks each push against verifier's key set as the +// relay does, as qumo rtmp's own authorizer with QUMO_AUTH_KEYS does. +func verifyingAuthorizer(verifier *auth.Verifier) ingest.PublishAuthorizer { + return func(credential string, path moqt.BroadcastPath) (ingest.Permission, error) { + g, err := verifier.Check(credential, token.ActionPublish, path) + if err != nil { + return ingest.Permission{}, err + } + if !g.Allows(token.ActionPublish, path) { + return ingest.Permission{}, fmt.Errorf("may not publish %s", path) + } + return ingest.Permission{Lapses: g.Expires(), KeyID: g.KeyID()}, nil + } +} + // TestABR_IngestOptsIn runs the opt-in through RTMP ingest, which opts a // stream in on its publisher's behalf (INGEST_OPT_IN): the publisher puts a // second token, for the opt-in, on its stream key, and the ingest announces @@ -193,16 +208,7 @@ func runIngestOptIn(t *testing.T, optIns []token.OptIn, paid bool) { // The ingest: it checks each publish against the key set as the relay // does, and publishes through the relay with the operator's key. - authorize := func(credential string, path moqt.BroadcastPath) (time.Time, error) { - g, err := verifier.Check(credential, token.ActionPublish, path) - if err != nil { - return time.Time{}, err - } - if !g.Allows(token.ActionPublish, path) { - return time.Time{}, fmt.Errorf("may not publish %s", path) - } - return g.Expires(), nil - } + authorize := verifyingAuthorizer(verifier) ingestMux := moqt.NewTrackMux(0) rtmpLn, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) diff --git a/internal/integration/optin_usage_test.go b/internal/integration/optin_usage_test.go index ce5f3571..af42dbc3 100644 --- a/internal/integration/optin_usage_test.go +++ b/internal/integration/optin_usage_test.go @@ -5,6 +5,8 @@ package integration import ( "context" "encoding/json" + "fmt" + "net" "net/http" "net/http/httptest" "os" @@ -18,6 +20,9 @@ import ( "github.com/stretchr/testify/require" "github.com/qumo-dev/qumo/internal/auth" + "github.com/qumo-dev/qumo/internal/ffpub" + "github.com/qumo-dev/qumo/internal/ingest" + "github.com/qumo-dev/qumo/internal/meter" "github.com/qumo-dev/qumo/internal/optin" "github.com/qumo-dev/qumo/internal/relay" "github.com/qumo-dev/qumo/token" @@ -133,3 +138,88 @@ func TestRelay_ReportsOptInSeconds(t *testing.T) { assert.LessOrEqual(t, secs, int64(5), "the time the opt-in lasted, not longer") assert.NoError(t, closed.Validate()) } + +// TestIngest_ReportsUsage pushes a stream over RTMP with two tokens, for the +// broadcast and for its ABR opt-in, to an ingest that checks them against a +// key set and reports usage. The stream's close record is under the tenant's +// key and carries the media the ingest took in and how long the opt-in +// lasted, so a control plane bills a stream pushed over RTMP as it bills one +// published over MoQ. +func TestIngest_ReportsUsage(t *testing.T) { + if !ffpub.Available() { + t.Skip("ffmpeg not on PATH") + } + tenant, err := token.GenerateKey("/live/") + require.NoError(t, err) + tenantPub := tenant.Public() + tenantPub.OptIns = []token.OptIn{token.OptInABR} + verifier := verifierFor(t, tenantPub) + sign := func(g token.Grant) string { + tok, err := token.Sign(tenant, g, time.Minute) + require.NoError(t, err) + return tok + } + sink := &usageSink{} + usageSrv := httptest.NewServer(sink) + t.Cleanup(usageSrv.Close) + m, err := meter.New(meter.Config{Name: "rtmp", URL: usageSrv.URL}) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + mux := moqt.NewTrackMux(0) + rtmpLn, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + rtmpAddr := rtmpLn.Addr().String() + require.NoError(t, rtmpLn.Close()) + go func() { + _ = ingest.NewRTMPServer(ingest.RTMPConfig{ + Addr: rtmpAddr, TrackMux: mux, Authorize: verifyingAuthorizer(verifier), + OptIns: []token.OptIn{token.OptInABR}, Meter: m, + }).ListenAndServe(ctx) + }() + require.Eventually(t, func() bool { + c, err := net.DialTimeout("tcp", rtmpAddr, 200*time.Millisecond) + if err == nil { + _ = c.Close() + } + return err == nil + }, 5*time.Second, 50*time.Millisecond) + + streamKey := fmt.Sprintf("metered?jwt=%s&jwt=%s", + sign(token.Grant{Root: "/live/", Publish: []string{"/metered"}}), + sign(token.Grant{Root: "/.abr/live/", Publish: []string{"/metered"}})) + pub := ffpub.New(ffpub.Config{ + URL: fmt.Sprintf("rtmp://%s/live/%s", rtmpAddr, streamKey), + GOP: 30, Width: 320, Height: 240, Framerate: 30, + }) + pubCtx, stopPub := context.WithCancel(ctx) + require.NoError(t, pub.Start(pubCtx)) + optInPath := optin.Path(token.ABRDirectory, "/live/metered") + require.Eventually(t, func() bool { + ann, _ := mux.TrackHandler(optInPath) + return ann != nil + }, 10*time.Second, 50*time.Millisecond, "the ingest did not opt the stream in") + + // Opted in for a little over two seconds, then the publisher stops. + time.Sleep(2200 * time.Millisecond) + stopPub() + _ = pub.Wait() + var closed *usage.Record + require.Eventually(t, func() bool { + require.NoError(t, m.Flush(ctx)) + for _, rec := range sink.all() { + if rec.Type == usage.SessionClose { + closed = &rec + } + } + return closed != nil + }, 10*time.Second, 100*time.Millisecond, "no session_close for the stream") + + assert.Equal(t, tenant.ID, closed.KID, "the tenant's key, from its token") + assert.Positive(t, closed.Metrics[usage.IngestMediaBytes], "the media the ingest took in") + secs := closed.Metrics[usage.OptInSeconds("abr")] + assert.GreaterOrEqual(t, secs, int64(2)) + assert.LessOrEqual(t, secs, int64(6)) + assert.NoError(t, closed.Validate()) +} diff --git a/internal/relay/optin_time.go b/internal/optin/clock.go similarity index 54% rename from internal/relay/optin_time.go rename to internal/optin/clock.go index 568b87e2..9d5d4242 100644 --- a/internal/relay/optin_time.go +++ b/internal/optin/clock.go @@ -1,4 +1,4 @@ -package relay +package optin import ( "sync" @@ -6,21 +6,17 @@ import ( "github.com/qumo-dev/gomoqt/moqt" - "github.com/qumo-dev/qumo/internal/optin" "github.com/qumo-dev/qumo/token" ) -// optInClock adds up how long a session keeps broadcasts opted into each -// egress: how long each announcement of its beneath an opt-in directory -// (/.abr/live/cam) lasts, summed over its announcements, so two broadcasts -// opted into ABR for a minute each are two minutes. The session's usage -// reports carry it (auth.Request.OptIns), for a control plane to bill an -// opt-in by the time it was used. A nil clock counts nothing. -// -// It counts what the relay received from the session: an announcement the -// session's grant does not cover never reaches it, so an opt-in the -// session's key does not hold is never counted. -type optInClock struct { +// A Clock adds up how long a publisher keeps broadcasts opted into each +// egress: how long each of its announcements beneath an opt-in directory +// (/.abr/live/cam) lasts, summed over them, so two broadcasts opted into ABR +// for a minute each are two minutes. A server's usage reports carry it +// (usage.OptInSeconds), for a control plane to bill an opt-in by the time it +// was used. A nil Clock counts nothing. Its methods are safe for concurrent +// use. +type Clock struct { now func() time.Time mu sync.Mutex @@ -35,19 +31,20 @@ type optInTally struct { since time.Time } -func newOptInClock(now func() time.Time) *optInClock { - return &optInClock{now: now, tallies: map[token.OptIn]*optInTally{}} +// NewClock returns a Clock reading the time from now (time.Now). +func NewClock(now func() time.Time) *Clock { + return &Clock{now: now, tallies: map[token.OptIn]*optInTally{}} } -// track counts ann, an announcement the session made, until it ends, when it +// Track counts ann, an announcement the publisher made, until it ends, when it // opts a broadcast into an egress; any other is not counted, nor one that // ended already (moqt runs AfterFunc for it later, on another goroutine, // which would count the time until then). -func (c *optInClock) track(ann *moqt.Announcement) { +func (c *Clock) Track(ann *moqt.Announcement) { if c == nil || !ann.IsActive() { return } - o, ok := optin.Of(ann.BroadcastPath()) + o, ok := Of(ann.BroadcastPath()) if !ok { return } @@ -57,7 +54,7 @@ func (c *optInClock) track(ann *moqt.Announcement) { // change adds delta to o's live announcements, counting the time they were // live until now. -func (c *optInClock) change(o token.OptIn, delta int) { +func (c *Clock) change(o token.OptIn, delta int) { c.mu.Lock() defer c.mu.Unlock() now := c.now() @@ -71,9 +68,9 @@ func (c *optInClock) change(o token.OptIn, delta int) { t.since = now } -// totals returns how long the session has kept broadcasts opted into each +// Totals returns how long the publisher has kept broadcasts opted into each // egress so far, live announcements counted up to now, or nil for none. -func (c *optInClock) totals() map[token.OptIn]time.Duration { +func (c *Clock) Totals() map[token.OptIn]time.Duration { if c == nil { return nil } diff --git a/internal/relay/optin_time_test.go b/internal/optin/clock_test.go similarity index 71% rename from internal/relay/optin_time_test.go rename to internal/optin/clock_test.go index be1cda27..ecf83793 100644 --- a/internal/relay/optin_time_test.go +++ b/internal/optin/clock_test.go @@ -1,4 +1,4 @@ -package relay +package optin import ( "context" @@ -25,26 +25,26 @@ func announce(t *testing.T, path moqt.BroadcastPath) (*moqt.Announcement, func() return ann, end } -func TestOptInClock(t *testing.T) { +func TestClock(t *testing.T) { clock := &fakeClock{t: time.Unix(1_700_000_000, 0)} - c := newOptInClock(clock.now) - assert.Nil(t, c.totals(), "nothing opted in yet") + c := NewClock(clock.now) + assert.Nil(t, c.Totals(), "nothing opted in yet") cam1, endCam1 := announce(t, "/.abr/live/cam1") cam2, endCam2 := announce(t, "/.abr/live/cam2") hls, _ := announce(t, "/.hls/live/cam1") plain, _ := announce(t, "/live/cam1") - c.track(cam1) - c.track(plain) + c.Track(cam1) + c.Track(plain) clock.add(10 * time.Second) - c.track(cam2) - c.track(hls) + c.Track(cam2) + c.Track(hls) clock.add(5 * time.Second) assert.Equal(t, map[token.OptIn]time.Duration{ token.OptInABR: 20 * time.Second, // cam1 15 s, cam2 5 s token.OptInHLS: 5 * time.Second, - }, c.totals(), "live announcements count up to now; a broadcast's own announcement is not an opt-in") + }, c.Totals(), "live announcements count up to now; a broadcast's own announcement is not an opt-in") endCam1() clock.add(10 * time.Second) @@ -54,24 +54,24 @@ func TestOptInClock(t *testing.T) { assert.Equal(t, map[token.OptIn]time.Duration{ token.OptInABR: 30 * time.Second, // cam1 0–15 s, cam2 10–25 s token.OptInHLS: time.Hour + 15*time.Second, - }, c.totals(), "an ended announcement stops counting") + }, c.Totals(), "an ended announcement stops counting") } -func TestOptInClock_EndedAnnouncement(t *testing.T) { +func TestClock_EndedAnnouncement(t *testing.T) { clock := &fakeClock{t: time.Unix(1_700_000_000, 0)} - c := newOptInClock(clock.now) + c := NewClock(clock.now) ann, end := announce(t, "/.abr/live/cam1") end() - c.track(ann) + c.Track(ann) clock.add(time.Minute) - assert.Nil(t, c.totals(), "an announcement that ended before it was tracked counts nothing") + assert.Nil(t, c.Totals(), "an announcement that ended before it was tracked counts nothing") } -func TestOptInClock_Nil(t *testing.T) { - var c *optInClock +func TestClock_Nil(t *testing.T) { + var c *Clock ann, _ := announce(t, "/.abr/live/cam1") - c.track(ann) - assert.Nil(t, c.totals(), "an unchecked session counts nothing") + c.Track(ann) + assert.Nil(t, c.Totals(), "a nil Clock counts nothing") } diff --git a/internal/relay/admit.go b/internal/relay/admit.go index 4030d9f8..36a4b052 100644 --- a/internal/relay/admit.go +++ b/internal/relay/admit.go @@ -15,6 +15,7 @@ import ( "github.com/qumo-dev/gomoqt/moqt" "github.com/qumo-dev/qumo/internal/auth" + "github.com/qumo-dev/qumo/internal/optin" "github.com/qumo-dev/qumo/token" "github.com/qumo-dev/qumo/usage" ) @@ -702,13 +703,13 @@ func (s *Server) admitClient(ctx context.Context, req auth.Request) (g *auth.Gra // ended. ctx is the session's: done by now, // so the report keeps its values but not its cancellation. A failed report is // recorded and otherwise ignored; End bounds its own wait. -func (s *Server) reportEnd(ctx context.Context, req auth.Request, stats moqt.SessionStats, optIns *optInClock, reason string) { +func (s *Server) reportEnd(ctx context.Context, req auth.Request, stats moqt.SessionStats, optIns *optin.Clock, reason string) { if s.End == nil { return } req.Event = auth.EventEnd req.Bytes = sessionBytes(stats) - req.OptIns = optIns.totals() + req.OptIns = optIns.Totals() req.Reason = reason if err := s.End(context.WithoutCancel(ctx), req); err != nil { metricAuthRequests.WithLabelValues(auth.EventEnd, authError).Inc() diff --git a/internal/relay/lease.go b/internal/relay/lease.go index 53d0ae12..c43fb324 100644 --- a/internal/relay/lease.go +++ b/internal/relay/lease.go @@ -13,6 +13,7 @@ import ( "github.com/qumo-dev/gomoqt/moqt" "github.com/qumo-dev/gomoqt/transport" "github.com/qumo-dev/qumo/internal/auth" + "github.com/qumo-dev/qumo/internal/optin" "github.com/qumo-dev/qumo/usage" ) @@ -61,7 +62,7 @@ type lease struct { req auth.Request // optIns is the session's opt-in time, sent with each revalidate; nil // for none. - optIns *optInClock + optIns *optin.Clock mu sync.Mutex timer *time.Timer @@ -77,7 +78,7 @@ type lease struct { // startLease starts the lease of a session admitted by req, to be revalidated // every cadence, reporting the opt-in time optIns counts (nil for none). It returns nil for a zero cadence, which never revalidates; // otherwise the caller stops it when the session ends first. -func startLease(ctx context.Context, sess leasedSession, authorize authorizeFunc, req auth.Request, cadence time.Duration, optIns *optInClock) *lease { +func startLease(ctx context.Context, sess leasedSession, authorize authorizeFunc, req auth.Request, cadence time.Duration, optIns *optin.Clock) *lease { if cadence <= 0 { return nil } @@ -171,7 +172,7 @@ func (l *lease) revalidate() (*auth.Grant, error) { req := l.req req.Event = auth.EventRevalidate req.Bytes = sessionBytes(l.sess.Stats()) - req.OptIns = l.optIns.totals() + req.OptIns = l.optIns.Totals() g, err := l.authorize(l.ctx, req) metricAuthRequests.WithLabelValues(auth.EventRevalidate, authOutcome(g, err)).Inc() return g, err diff --git a/internal/relay/server.go b/internal/relay/server.go index 0181cee8..e634c520 100644 --- a/internal/relay/server.go +++ b/internal/relay/server.go @@ -14,6 +14,7 @@ import ( "github.com/qumo-dev/gomoqt/moqt" "github.com/qumo-dev/qumo/internal/auth" "github.com/qumo-dev/qumo/internal/cors" + "github.com/qumo-dev/qumo/internal/optin" ) type Server struct { @@ -811,9 +812,9 @@ func (s *Server) serveSession(sess *moqt.Session) { var l *lease // optIns counts the opt-in time of a checked session, whose usage is // reported; nil counts none. - var optIns *optInClock + var optIns *optin.Clock if checked { - optIns = newOptInClock(time.Now) + optIns = optin.NewClock(time.Now) a.served.Store(true) // Registered first so that it runs last: after the close below, // with the session's final byte totals and close cause. Counted in @@ -878,7 +879,7 @@ func (s *Server) serveSession(sess *moqt.Session) { return } - optIns.track(ann) + optIns.Track(ann) slog.Debug("relay: received announcement", "node", s.Config.NodeID, diff --git a/usage/usage.go b/usage/usage.go index d434a585..1137cecb 100644 --- a/usage/usage.go +++ b/usage/usage.go @@ -13,6 +13,11 @@ // opted into each egress (OptInSeconds), so that a control plane can bill an // option such as ABR by the time it was used. // +// An RTMP or RTSP ingest sends the same records for each stream pushed to +// it, under the key of the publisher's token: SessionOpen, Usage and +// SessionClose with the media it took in (IngestMediaBytes) and its opt-in +// time. +// // Byte counts are cumulative and from the relay's point of view (what it // received is ingress), so a receiver can take a resent record without // counting it twice. A relay that restarts begins a new Run, whose viewer @@ -50,9 +55,13 @@ const ( EgressBytes = "gateway.egress_bytes" ) -// OptInSeconds names the metric of how long a publishing session kept -// broadcasts opted into the egress named optIn ("hls", "abr"), in whole -// seconds, added up over its broadcasts: two broadcasts opted into ABR for +// IngestMediaBytes is the media an RTMP or RTSP ingest took in from a +// publisher: the payload of every frame, without the protocol around it. +const IngestMediaBytes = "ingest.media_bytes" + +// OptInSeconds names the metric of how long a publishing session, on a relay +// or an ingest, kept broadcasts opted into the egress named optIn ("hls", +// "abr"), in whole seconds, added up over its broadcasts: two broadcasts opted into ABR for // a minute each are 120. A session that opted none in carries none. // // OptInSeconds("abr") is "gateway.opt_in_seconds.abr" From ad52c90c281429574b75ee2ebc99c06ed65ab9c5 Mon Sep 17 00:00:00 2001 From: daichiDeskTop Date: Wed, 18 Nov 2026 21:22:56 +0900 Subject: [PATCH 6/7] fix(relay): send no usage record while a publisher has nothing to report A publisher that opted anything in sent a usage record at every revalidate, even with no bytes moved and under a whole second opted in. It now reports only when a metric is positive, as before #613 for bytes. The opt-in clock reads time.Now itself; its tests run in a synctest bubble instead of passing a fake clock through the constructor. TestOf uses subtests; two doc comments are rewrapped. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KPWMJNAzsnsiqRPREa2JqY --- internal/auth/usage.go | 18 +++++-- internal/auth/usage_viewers_test.go | 12 +++-- internal/optin/optin_test.go | 8 ++-- internal/relay/admit.go | 6 +-- internal/relay/lease.go | 5 +- internal/relay/optin_time.go | 10 ++-- internal/relay/optin_time_test.go | 73 ++++++++++++++--------------- internal/relay/server.go | 2 +- 8 files changed, 74 insertions(+), 60 deletions(-) diff --git a/internal/auth/usage.go b/internal/auth/usage.go index 7c54638b..18cc9795 100644 --- a/internal/auth/usage.go +++ b/internal/auth/usage.go @@ -117,11 +117,10 @@ func (r *usageReporter) revalidated(id string, b Bytes, optIns map[token.OptIn]t if live { s.seen = r.now() } - switch { - case !s.publishes: + if !s.publishes { s = r.countViewer(s, b) - case b.Sent > s.counted.Sent || b.Received > s.counted.Received || len(optIns) > 0: - r.meter.Report(id, s.kid, publisherMetrics(b.since(s.counted), optIns)) + } else if m := publisherMetrics(b.since(s.counted), optIns); carriesUsage(m) { + r.meter.Report(id, s.kid, m) } r.sessions[id] = s } @@ -179,6 +178,17 @@ func publisherMetrics(b Bytes, optIns map[token.OptIn]time.Duration) map[string] return m } +// carriesUsage reports whether m holds any usage: a session that moved +// nothing and kept nothing opted in for a whole second has none to report. +func carriesUsage(m map[string]int64) bool { + for _, v := range m { + if v > 0 { + return true + } + } + return false +} + func clampInt64(v uint64) int64 { if v > math.MaxInt64 { return math.MaxInt64 diff --git a/internal/auth/usage_viewers_test.go b/internal/auth/usage_viewers_test.go index 5775422a..ac7459d3 100644 --- a/internal/auth/usage_viewers_test.go +++ b/internal/auth/usage_viewers_test.go @@ -158,17 +158,23 @@ func TestUsageReporter_OptInSeconds(t *testing.T) { assert.NotContains(t, first[usage.Viewers].Metrics, usage.OptInSeconds("abr"), "a viewer opts nothing in") r.open("p2", usageSession{kid: "k1", publishes: true}) - r.revalidated("p2", Bytes{}, abr(2*time.Second), true) + r.revalidated("p2", Bytes{}, abr(400*time.Millisecond), true) require.NoError(t, r.meter.Flush(context.Background())) require.Len(t, sink.received(), 2) - assert.Equal(t, int64(2), byType(sink.received()[1])[usage.Usage].Metrics[usage.OptInSeconds("abr")], + assert.NotContains(t, byType(sink.received()[1]), usage.Usage, + "no bytes and under a second opted in is nothing to report") + + r.revalidated("p2", Bytes{}, abr(2*time.Second), true) + require.NoError(t, r.meter.Flush(context.Background())) + require.Len(t, sink.received(), 3) + assert.Equal(t, int64(2), byType(sink.received()[2])[usage.Usage].Metrics[usage.OptInSeconds("abr")], "opt-in time without bytes is reported") assert.True(t, r.close("p1", Bytes{Received: 120}, map[token.OptIn]time.Duration{ token.OptInABR: 3 * time.Second, token.OptInHLS: 500 * time.Millisecond, }, usage.ReasonClosed)) require.NoError(t, r.meter.Flush(context.Background())) - closed := byType(sink.received()[2])[usage.SessionClose] + closed := byType(sink.received()[3])[usage.SessionClose] assert.Equal(t, map[string]int64{ usage.IngressBytes: 120, usage.EgressBytes: 0, usage.OptInSeconds("abr"): 3, }, closed.Metrics, "under a second of HLS is left out") diff --git a/internal/optin/optin_test.go b/internal/optin/optin_test.go index 2c7ae399..3cc66b98 100644 --- a/internal/optin/optin_test.go +++ b/internal/optin/optin_test.go @@ -54,9 +54,11 @@ func TestOf(t *testing.T) { "/live/.abr/cam1": {}, } for p, tt := range tests { - got, ok := Of(p) - assert.Equal(t, tt.wantOK, ok, p) - assert.Equal(t, tt.want, got, p) + t.Run(string(p), func(t *testing.T) { + got, ok := Of(p) + assert.Equal(t, tt.wantOK, ok) + assert.Equal(t, tt.want, got) + }) } } diff --git a/internal/relay/admit.go b/internal/relay/admit.go index 4030d9f8..afbbe852 100644 --- a/internal/relay/admit.go +++ b/internal/relay/admit.go @@ -699,9 +699,9 @@ func (s *Server) admitClient(ctx context.Context, req auth.Request) (g *auth.Gra // reportEnd sends the end event of a checked session admitted by req, with // its final byte totals, its opt-in time (optIns, nil for none) and why it -// ended. ctx is the session's: done by now, -// so the report keeps its values but not its cancellation. A failed report is -// recorded and otherwise ignored; End bounds its own wait. +// ended. ctx is the session's: done by now, so the report keeps its values +// but not its cancellation. A failed report is recorded and otherwise +// ignored; End bounds its own wait. func (s *Server) reportEnd(ctx context.Context, req auth.Request, stats moqt.SessionStats, optIns *optInClock, reason string) { if s.End == nil { return diff --git a/internal/relay/lease.go b/internal/relay/lease.go index 53d0ae12..a63c70fd 100644 --- a/internal/relay/lease.go +++ b/internal/relay/lease.go @@ -75,8 +75,9 @@ type lease struct { } // startLease starts the lease of a session admitted by req, to be revalidated -// every cadence, reporting the opt-in time optIns counts (nil for none). It returns nil for a zero cadence, which never revalidates; -// otherwise the caller stops it when the session ends first. +// every cadence, reporting the opt-in time optIns counts (nil for none). It +// returns nil for a zero cadence, which never revalidates; otherwise the +// caller stops it when the session ends first. func startLease(ctx context.Context, sess leasedSession, authorize authorizeFunc, req auth.Request, cadence time.Duration, optIns *optInClock) *lease { if cadence <= 0 { return nil diff --git a/internal/relay/optin_time.go b/internal/relay/optin_time.go index 568b87e2..4e4834af 100644 --- a/internal/relay/optin_time.go +++ b/internal/relay/optin_time.go @@ -21,8 +21,6 @@ import ( // session's grant does not cover never reaches it, so an opt-in the // session's key does not hold is never counted. type optInClock struct { - now func() time.Time - mu sync.Mutex tallies map[token.OptIn]*optInTally } @@ -35,8 +33,8 @@ type optInTally struct { since time.Time } -func newOptInClock(now func() time.Time) *optInClock { - return &optInClock{now: now, tallies: map[token.OptIn]*optInTally{}} +func newOptInClock() *optInClock { + return &optInClock{tallies: map[token.OptIn]*optInTally{}} } // track counts ann, an announcement the session made, until it ends, when it @@ -60,7 +58,7 @@ func (c *optInClock) track(ann *moqt.Announcement) { func (c *optInClock) change(o token.OptIn, delta int) { c.mu.Lock() defer c.mu.Unlock() - now := c.now() + now := time.Now() t := c.tallies[o] if t == nil { t = &optInTally{} @@ -82,7 +80,7 @@ func (c *optInClock) totals() map[token.OptIn]time.Duration { if len(c.tallies) == 0 { return nil } - now := c.now() + now := time.Now() out := make(map[token.OptIn]time.Duration, len(c.tallies)) for o, t := range c.tallies { out[o] = t.done + time.Duration(t.live)*now.Sub(t.since) diff --git a/internal/relay/optin_time_test.go b/internal/relay/optin_time_test.go index be1cda27..9fcf3786 100644 --- a/internal/relay/optin_time_test.go +++ b/internal/relay/optin_time_test.go @@ -3,6 +3,7 @@ package relay import ( "context" "testing" + "testing/synctest" "time" "github.com/qumo-dev/gomoqt/moqt" @@ -11,12 +12,6 @@ import ( "github.com/qumo-dev/qumo/token" ) -// fakeClock is a time that moves only when told. -type fakeClock struct{ t time.Time } - -func (c *fakeClock) now() time.Time { return c.t } -func (c *fakeClock) add(d time.Duration) { c.t = c.t.Add(d) } - // announce returns a live announcement of path and the function that ends it. func announce(t *testing.T, path moqt.BroadcastPath) (*moqt.Announcement, func()) { t.Helper() @@ -26,47 +21,49 @@ func announce(t *testing.T, path moqt.BroadcastPath) (*moqt.Announcement, func() } func TestOptInClock(t *testing.T) { - clock := &fakeClock{t: time.Unix(1_700_000_000, 0)} - c := newOptInClock(clock.now) - assert.Nil(t, c.totals(), "nothing opted in yet") + synctest.Test(t, func(t *testing.T) { + c := newOptInClock() + assert.Nil(t, c.totals(), "nothing opted in yet") - cam1, endCam1 := announce(t, "/.abr/live/cam1") - cam2, endCam2 := announce(t, "/.abr/live/cam2") - hls, _ := announce(t, "/.hls/live/cam1") - plain, _ := announce(t, "/live/cam1") - c.track(cam1) - c.track(plain) - clock.add(10 * time.Second) - c.track(cam2) - c.track(hls) - clock.add(5 * time.Second) + cam1, endCam1 := announce(t, "/.abr/live/cam1") + cam2, endCam2 := announce(t, "/.abr/live/cam2") + hls, _ := announce(t, "/.hls/live/cam1") + plain, _ := announce(t, "/live/cam1") + c.track(cam1) + c.track(plain) + time.Sleep(10 * time.Second) + c.track(cam2) + c.track(hls) + time.Sleep(5 * time.Second) - assert.Equal(t, map[token.OptIn]time.Duration{ - token.OptInABR: 20 * time.Second, // cam1 15 s, cam2 5 s - token.OptInHLS: 5 * time.Second, - }, c.totals(), "live announcements count up to now; a broadcast's own announcement is not an opt-in") + assert.Equal(t, map[token.OptIn]time.Duration{ + token.OptInABR: 20 * time.Second, // cam1 15 s, cam2 5 s + token.OptInHLS: 5 * time.Second, + }, c.totals(), "live announcements count up to now; a broadcast's own announcement is not an opt-in") - endCam1() - clock.add(10 * time.Second) - endCam2() - clock.add(time.Hour) + endCam1() + time.Sleep(10 * time.Second) + endCam2() + time.Sleep(time.Hour) - assert.Equal(t, map[token.OptIn]time.Duration{ - token.OptInABR: 30 * time.Second, // cam1 0–15 s, cam2 10–25 s - token.OptInHLS: time.Hour + 15*time.Second, - }, c.totals(), "an ended announcement stops counting") + assert.Equal(t, map[token.OptIn]time.Duration{ + token.OptInABR: 30 * time.Second, // cam1 0–15 s, cam2 10–25 s + token.OptInHLS: time.Hour + 15*time.Second, + }, c.totals(), "an ended announcement stops counting") + }) } func TestOptInClock_EndedAnnouncement(t *testing.T) { - clock := &fakeClock{t: time.Unix(1_700_000_000, 0)} - c := newOptInClock(clock.now) - ann, end := announce(t, "/.abr/live/cam1") - end() + synctest.Test(t, func(t *testing.T) { + c := newOptInClock() + ann, end := announce(t, "/.abr/live/cam1") + end() - c.track(ann) - clock.add(time.Minute) + c.track(ann) + time.Sleep(time.Minute) - assert.Nil(t, c.totals(), "an announcement that ended before it was tracked counts nothing") + assert.Nil(t, c.totals(), "an announcement that ended before it was tracked counts nothing") + }) } func TestOptInClock_Nil(t *testing.T) { diff --git a/internal/relay/server.go b/internal/relay/server.go index 0181cee8..c33ecc5f 100644 --- a/internal/relay/server.go +++ b/internal/relay/server.go @@ -813,7 +813,7 @@ func (s *Server) serveSession(sess *moqt.Session) { // reported; nil counts none. var optIns *optInClock if checked { - optIns = newOptInClock(time.Now) + optIns = newOptInClock() a.served.Store(true) // Registered first so that it runs last: after the close below, // with the session's final byte totals and close cause. Counted in From 7b067c88b5e6fb21e2015a305fea9452d4a0fd7d Mon Sep 17 00:00:00 2001 From: daichiDeskTop Date: Wed, 18 Nov 2026 21:26:46 +0900 Subject: [PATCH 7/7] fix(ingest): wait for the stream's open in the RTMP meter test, and share the whole-seconds rule TestRTMPServer_Meter flushed the meter once the broadcast was announced, but the stream's usage report opens after that, so the sink often got nothing. It now flushes until the open arrives. TestMeterStream runs its stream in a synctest bubble instead of sleeping for real time. The rule turning opt-in time into whole-second metrics was written in both auth and ingest; optin.AddSeconds is now the one copy. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KPWMJNAzsnsiqRPREa2JqY --- internal/auth/usage.go | 7 ++---- internal/ingest/usage.go | 7 ++---- internal/ingest/usage_test.go | 42 +++++++++++++++++++++++------------ internal/optin/clock.go | 12 ++++++++++ 4 files changed, 44 insertions(+), 24 deletions(-) diff --git a/internal/auth/usage.go b/internal/auth/usage.go index 7c54638b..97a1a61c 100644 --- a/internal/auth/usage.go +++ b/internal/auth/usage.go @@ -7,6 +7,7 @@ import ( "time" "github.com/qumo-dev/qumo/internal/meter" + "github.com/qumo-dev/qumo/internal/optin" "github.com/qumo-dev/qumo/internal/reqsign" "github.com/qumo-dev/qumo/token" "github.com/qumo-dev/qumo/usage" @@ -171,11 +172,7 @@ func usageMetrics(b Bytes) map[string]int64 { // long it kept broadcasts opted into each egress, in whole seconds. func publisherMetrics(b Bytes, optIns map[token.OptIn]time.Duration) map[string]int64 { m := usageMetrics(b) - for o, d := range optIns { - if secs := int64(d / time.Second); secs > 0 { - m[usage.OptInSeconds(string(o))] = secs - } - } + optin.AddSeconds(m, optIns) return m } diff --git a/internal/ingest/usage.go b/internal/ingest/usage.go index adc9e498..686fc638 100644 --- a/internal/ingest/usage.go +++ b/internal/ingest/usage.go @@ -7,6 +7,7 @@ import ( "time" "github.com/qumo-dev/qumo/internal/meter" + "github.com/qumo-dev/qumo/internal/optin" "github.com/qumo-dev/qumo/usage" ) @@ -51,10 +52,6 @@ func meterStream(m *meter.Reporter, kid string, sess *Session) (end func(reason func streamMetrics(sess *Session) map[string]int64 { mediaBytes, optIns := sess.Usage() m := map[string]int64{usage.IngestMediaBytes: int64(min(mediaBytes, math.MaxInt64))} - for o, d := range optIns { - if secs := int64(d / time.Second); secs > 0 { - m[usage.OptInSeconds(string(o))] = secs - } - } + optin.AddSeconds(m, optIns) return m } diff --git a/internal/ingest/usage_test.go b/internal/ingest/usage_test.go index 249c5378..cc68f069 100644 --- a/internal/ingest/usage_test.go +++ b/internal/ingest/usage_test.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" "sync" "testing" + "testing/synctest" "time" "github.com/qumo-dev/gomoqt/moqt" @@ -55,15 +56,19 @@ func newTestMeter(t *testing.T) (*meter.Reporter, *usageSink) { // open, and a close carrying the media it took in and its opt-in time. func TestMeterStream(t *testing.T) { m, sink := newTestMeter(t) - sess, err := NewSession(moqt.NewTrackMux(0), "/live/cam") - require.NoError(t, err) - defer sess.Close() - - end := meterStream(m, "k1", sess) - sess.PushVideo(0, make([]byte, 500), true) - sess.OptIn(token.OptInABR, time.Time{}) - time.Sleep(1100 * time.Millisecond) - end(usage.ReasonClosed) + // The stream runs in a bubble, so its opt-in lasts exactly as long as + // the sleep; the meter sends over the network outside it. + synctest.Test(t, func(t *testing.T) { + sess, err := NewSession(moqt.NewTrackMux(0), "/live/cam") + require.NoError(t, err) + defer sess.Close() + + end := meterStream(m, "k1", sess) + sess.PushVideo(0, make([]byte, 500), true) + sess.OptIn(token.OptInABR, time.Time{}) + time.Sleep(1500 * time.Millisecond) + end(usage.ReasonClosed) + }) require.NoError(t, m.Flush(t.Context())) records := sink.all() @@ -113,11 +118,20 @@ func TestRTMPServer_Meter(t *testing.T) { publishRTMP(t, addr, "live", "refused?jwt=nope") publishRTMP(t, addr, "live", "cam?jwt=tok") - require.Eventually(t, func() bool { return published(mux, "/live/cam") }, 3*time.Second, 10*time.Millisecond) - require.NoError(t, m.Flush(t.Context())) + // The stream is announced before its usage report opens, so wait for + // the open itself. + var records []usage.Record + require.Eventually(t, func() bool { + if err := m.Flush(t.Context()); err != nil { + return false + } + records = append(records, sink.all()[len(records):]...) + return len(records) > 0 + }, 3*time.Second, 10*time.Millisecond) - records := sink.all() - require.Len(t, records, 1, "an open for the stream granted; none for the refused one") assert.Equal(t, usage.SessionOpen, records[0].Type) - assert.Equal(t, "tenant-key", records[0].KID) + for _, rec := range records { + assert.Equal(t, "tenant-key", rec.KID, "only the stream granted is reported, under the key that granted it") + assert.Equal(t, records[0].SessionID, rec.SessionID) + } } diff --git a/internal/optin/clock.go b/internal/optin/clock.go index 9d5d4242..e957dd56 100644 --- a/internal/optin/clock.go +++ b/internal/optin/clock.go @@ -7,6 +7,7 @@ import ( "github.com/qumo-dev/gomoqt/moqt" "github.com/qumo-dev/qumo/token" + "github.com/qumo-dev/qumo/usage" ) // A Clock adds up how long a publisher keeps broadcasts opted into each @@ -68,6 +69,17 @@ func (c *Clock) change(o token.OptIn, delta int) { t.since = now } +// AddSeconds adds each egress's time in optIns to m, as Totals returns it, +// in whole seconds under usage.OptInSeconds. An egress with less than a +// second is left out, so a session that opted nothing in carries no metric. +func AddSeconds(m map[string]int64, optIns map[token.OptIn]time.Duration) { + for o, d := range optIns { + if secs := int64(d / time.Second); secs > 0 { + m[usage.OptInSeconds(string(o))] = secs + } + } +} + // Totals returns how long the publisher has kept broadcasts opted into each // egress so far, live announcements counted up to now, or nil for none. func (c *Clock) Totals() map[token.OptIn]time.Duration {