From 8eb9443b4b6590e78b21bf384ca7007e2042be30 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 11:54:35 +0000 Subject: [PATCH 1/2] feat(ingest): report each pushed stream's usage under its publisher's key With QUMO_USAGE_URL and QUMO_AUTH_KEYS, qumo rtmp and qumo rtsp-push send each stream to the usage URL as a relay sends a publishing session: an open, usage every 30 s and a close, under the kid of the token that granted the push, with the media the ingest took in (ingest.media_bytes) and how long it kept the broadcast opted into each egress. The relay only sees the ingest's own session, so a stream pushed over RTMP or RTSP is billed to its publisher here. - ingest.PublishAuthorizer returns a Permission: the lapse, and the kid of the key that granted it. - Session counts the media pushed into it and its opt-in time (Usage). - The opt-in clock moved from the relay to internal/optin (optin.Clock), which both use. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KPWMJL7PH3rPhK5UTGavaB --- CHANGELOG.md | 1 + docs/site/content/en/docs/cli/rtmp.md | 14 +- docs/site/content/en/docs/cli/rtsp-push.md | 14 +- docs/site/content/en/docs/configuration.md | 2 +- internal/ingest/auth.go | 30 +++-- internal/ingest/auth_test.go | 18 ++- internal/ingest/cmd.go | 84 +++++++++--- .../ingest/fake_publish_authorizer_test.go | 12 +- internal/ingest/rtmp.go | 17 ++- internal/ingest/rtmp_server_test.go | 15 ++- internal/ingest/rtsp.go | 19 ++- internal/ingest/rtsp_test.go | 6 +- internal/ingest/session.go | 17 +++ internal/ingest/session_test.go | 20 +++ internal/ingest/usage.go | 60 +++++++++ internal/ingest/usage_test.go | 123 ++++++++++++++++++ internal/integration/abr_auth_test.go | 26 ++-- internal/integration/optin_usage_test.go | 90 +++++++++++++ .../{relay/optin_time.go => optin/clock.go} | 39 +++--- .../clock_test.go} | 36 ++--- internal/relay/admit.go | 5 +- internal/relay/lease.go | 7 +- internal/relay/server.go | 7 +- usage/usage.go | 15 ++- 24 files changed, 563 insertions(+), 114 deletions(-) create mode 100644 internal/ingest/usage.go create mode 100644 internal/ingest/usage_test.go rename internal/{relay/optin_time.go => optin/clock.go} (54%) rename internal/{relay/optin_time_test.go => optin/clock_test.go} (71%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 430cb6d1..2f99cfdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **`PEERS` names are resolved again while the relay runs, so peers behind a name join and leave without a restart; an entry without a port is an SRV name (`_moqt._udp.hubs.example.com`) (`internal/relay`).** A `PEERS` entry that is neither `host:port` nor a name without a port now stops the relay at startup, and a host that doesn't resolve is retried instead of being dialed by name. - **The relay re-reads its certificates while it runs (`internal/relay`).** About once a minute it reads `CERT_FILE`/`KEY_FILE`, `PEER_CERT_FILE`/`PEER_KEY_FILE` and `CA_FILE` and compares them with what it holds; what changed is taken from the next TLS handshake on, so cert-manager, a Vault or step-ca agent, or any job that writes the files needs no restart. A public certificate is taken once it parses and matches its key; a peer certificate and the CA together, once they pass the startup checks (issued by the CA, the peering name, valid now). Anything else is logged, once per distinct error rather than at every read, and the certificates in use are kept; renewal tools should replace the files atomically (write, then rename). Sessions already established are untouched. The CA is now checked at each handshake on both sides: as the dialed relay (`GetConfigForClient`) and as the dialer (`VerifyConnection` against the current CA, with the same checks crypto/tls made). The relay's key-set and usage requests are signed with the renewed peer certificate (`reqsign.Signer.Rotate`), and `qumo dns` re-reads its `PEER_CERT_FILE`/`PEER_KEY_FILE` on the same one-minute cadence and signs its feed reads with a renewed pair (`internal/certreload`, shared by both). New metrics: `qumo_relay_cert_reloads_total{cert,result}` and `qumo_relay_cert_not_after_seconds{cert}`, and `qumo_dns_cert_reloads_total{cert,result}` and `qumo_dns_cert_not_after_seconds{cert}`; alert on the `not_after` gauges. A CA is rotated by putting the new one beside the old in `CA_FILE` first ([TLS & mTLS](docs/site/content/en/docs/deployment/tls.md#certificate-renewal)). - **A relay reports how long each publishing session kept broadcasts opted into HLS and ABR (`internal/relay`, `usage.OptInSeconds`).** A publishing session's `usage` and `session_close` records now carry `gateway.opt_in_seconds.abr` and `gateway.opt_in_seconds.hls`: the whole seconds its announcements beneath `/.abr/` and `/.hls/` lasted, added up over its broadcasts, from when the relay received each until it ended, the session closed or a revoked ticket ended it. Only an opt-in the session's grant covers reaches the relay, so the time is billed to the key that holds the opt-in, and a control plane can charge for ABR or HLS by the time it was used. `qumo abr` itself never needs to know whose broadcast it transcodes. A session that opted nothing into an egress carries no metric for it, so a receiver that doesn't bill opt-ins sees records as before. +- **RTMP and RTSP ingest report each pushed stream's usage under its publisher's key (`internal/ingest`, `QUMO_USAGE_URL`).** With `QUMO_USAGE_URL` and `QUMO_AUTH_KEYS`, `qumo rtmp` and `qumo rtsp-push` send each stream to the usage URL as a relay sends a publishing session (`internal/meter`): a `session_open`, a `usage` record every 30 s and a `session_close`, under the kid of the token that granted the push, carrying the media the ingest took in (`usage.IngestMediaBytes`, `ingest.media_bytes`: the payload of every frame) and how long it kept the broadcast opted into each egress (`gateway.opt_in_seconds.*`). The relay sees only the ingest's own session, under the ingest's key, so a stream pushed over RTMP or RTSP is billed to its publisher here, as one published over MoQ is billed on the relay. `ingest.PublishAuthorizer` now returns a `Permission`, the lapse and the kid of the key that granted it. The opt-in clock moved from the relay to `internal/optin` (`optin.Clock`), which both use. ### Changed diff --git a/docs/site/content/en/docs/cli/rtmp.md b/docs/site/content/en/docs/cli/rtmp.md index 34fbc5f6..4ac41384 100644 --- a/docs/site/content/en/docs/cli/rtmp.md +++ b/docs/site/content/en/docs/cli/rtmp.md @@ -90,6 +90,17 @@ listener checks no subscriber, so with auth on, viewers go through the relay. $ RELAY_URL=moqt://relay:4433 RELAY_SIGNING_KEY=ingest-key.jwk RELAY_PUBLISH=/live/ qumo rtmp ``` +## Usage + +With `QUMO_USAGE_URL` (and `QUMO_AUTH_KEYS`), the server reports each pushed +stream as a relay reports a publishing session ([usage reports]({{< relref "../configuration#usage-reports" >}})), +under the key of the token that granted the push: a `session_open`, a `usage` +record every 30 s and a `session_close`, carrying the media it took in +(`ingest.media_bytes`, the payload of every frame) and how long it kept its +broadcast opted into each egress (`gateway.opt_in_seconds.abr`, +`gateway.opt_in_seconds.hls`). The relay sees the server's own session, under +the server's key, so the stream is billed here, to its publisher. + ## Configuration | Variable | Default | Description | @@ -97,7 +108,8 @@ $ RELAY_URL=moqt://relay:4433 RELAY_SIGNING_KEY=ingest-key.jwk RELAY_PUBLISH=/li | `RTMP_INGEST_ADDR` | `:1935` | RTMP listen address. | | `RTMP_SERVE_ADDR` | `:4433`; none with `RELAY_URL` | MoQT listen address for subscribers to dial directly. | | `QUMO_AUTH_KEYS` | (unset) | The key set a publisher's credential is verified against, as the relay reads it (a file or URL). Unset accepts every publish. | -| `QUMO_AUTH_KEYS_CACHE` / `QUMO_RELAY_TOKEN` | (unset) | A file a downloaded key set is kept in, and the bearer token sent to a key-set URL. | +| `QUMO_AUTH_KEYS_CACHE` / `QUMO_RELAY_TOKEN` | (unset) | A file a downloaded key set is kept in, and the bearer token sent to a key-set URL and to `QUMO_USAGE_URL`. | +| `QUMO_USAGE_URL` | (unset) | Where each stream's usage is reported, under the key of the publisher's token ([usage](#usage)). Needs `QUMO_AUTH_KEYS`. | | `INGEST_OPT_IN` | (unset) | The egresses (`hls`, `abr`) a stream is opted into on its publisher's behalf, comma-separated: with auth on, those a second `?jwt=` may publish the opt-in of. | | `RELAY_URL` | (unset) | The relay to publish through, dialed as a client (`https://…` or `moqt://…`); it may carry a credential as `?jwt=`. | | `RELAY_SIGNING_KEY`, `RELAY_PUBLISH` | (unset) | Set together: a signing key (`qumo auth keygen`) the server signs a fresh relay credential with on every dial and renews on the session, granting publishing beneath the directory `RELAY_PUBLISH`. A push outside it is refused. | diff --git a/docs/site/content/en/docs/cli/rtsp-push.md b/docs/site/content/en/docs/cli/rtsp-push.md index 0e626e10..d7d9a0f9 100644 --- a/docs/site/content/en/docs/cli/rtsp-push.md +++ b/docs/site/content/en/docs/cli/rtsp-push.md @@ -89,6 +89,17 @@ listener checks no subscriber, so with auth on, viewers go through the relay. $ RELAY_URL=moqt://relay:4433 RELAY_SIGNING_KEY=ingest-key.jwk RELAY_PUBLISH=/live/ qumo rtsp-push ``` +## Usage + +With `QUMO_USAGE_URL` (and `QUMO_AUTH_KEYS`), the server reports each pushed +stream as a relay reports a publishing session ([usage reports]({{< relref "../configuration#usage-reports" >}})), +under the key of the token that granted the push: a `session_open`, a `usage` +record every 30 s and a `session_close`, carrying the media it took in +(`ingest.media_bytes`, the payload of every frame) and how long it kept its +broadcast opted into each egress (`gateway.opt_in_seconds.abr`, +`gateway.opt_in_seconds.hls`). The relay sees the server's own session, under +the server's key, so the stream is billed here, to its publisher. + ## Configuration | Variable | Default | Description | @@ -96,7 +107,8 @@ $ RELAY_URL=moqt://relay:4433 RELAY_SIGNING_KEY=ingest-key.jwk RELAY_PUBLISH=/li | `RTSP_INGEST_ADDR` | `:8554` | RTSP listen address. | | `RTSP_SERVE_ADDR` | `:4433`; none with `RELAY_URL` | MoQT listen address for subscribers to dial directly. | | `QUMO_AUTH_KEYS` | (unset) | The key set a publisher's credential is verified against, as the relay reads it (a file or URL). Unset accepts every publish. | -| `QUMO_AUTH_KEYS_CACHE` / `QUMO_RELAY_TOKEN` | (unset) | A file a downloaded key set is kept in, and the bearer token sent to a key-set URL. | +| `QUMO_AUTH_KEYS_CACHE` / `QUMO_RELAY_TOKEN` | (unset) | A file a downloaded key set is kept in, and the bearer token sent to a key-set URL and to `QUMO_USAGE_URL`. | +| `QUMO_USAGE_URL` | (unset) | Where each stream's usage is reported, under the key of the publisher's token ([usage](#usage)). Needs `QUMO_AUTH_KEYS`. | | `INGEST_OPT_IN` | (unset) | The egresses (`hls`, `abr`) a stream is opted into on its publisher's behalf, comma-separated: with auth on, those a second `?jwt=` may publish the opt-in of. | | `RELAY_URL` | (unset) | The relay to publish through, dialed as a client (`https://…` or `moqt://…`); it may carry a credential as `?jwt=`. | | `RELAY_SIGNING_KEY`, `RELAY_PUBLISH` | (unset) | Set together: a signing key (`qumo auth keygen`) the server signs a fresh relay credential with on every dial and renews on the session, granting publishing beneath the directory `RELAY_PUBLISH`. A push outside it is refused. | diff --git a/docs/site/content/en/docs/configuration.md b/docs/site/content/en/docs/configuration.md index 5a8dfed3..f1a5dc7e 100644 --- a/docs/site/content/en/docs/configuration.md +++ b/docs/site/content/en/docs/configuration.md @@ -265,7 +265,7 @@ Byte counts are cumulative and from the relay's point of view (what it received **Sessions that publish a broadcast are reported one by one,** as the first three records: an open, usage every 30 s, and a close. A session publishes when its token has a `publish` pattern, or from when it is granted a ticket that has one: its open comes then, and what it moved as a viewer up to its last 30 s check stays in its key's viewer total, the rest counting as its usage. -**A publishing session's records also carry its opt-in time:** how long it kept broadcasts opted into each egress, `gateway.opt_in_seconds.abr` for [`qumo abr`](../cli/abr/) and `gateway.opt_in_seconds.hls` for [`qumo hls`](../cli/hls/), in whole seconds, added up over its broadcasts (two broadcasts opted into ABR for a minute each are 120). It is the time each announcement of the session beneath `/.abr/` or `/.hls/` lasted on this relay, from when the relay received it until it ended, the session closed or a revoked ticket ended it, so a control plane can bill an opt-in by the time it was used. Only an opt-in the session's grant covers reaches the relay, so an opt-in a key doesn't hold is never counted. An egress the session opted nothing into has no metric. A broadcast opted in through [RTMP or RTSP ingest](../cli/rtmp/#opting-streams-into-hls-and-abr) counts on the ingest's session, not the publisher's. +**A publishing session's records also carry its opt-in time:** how long it kept broadcasts opted into each egress, `gateway.opt_in_seconds.abr` for [`qumo abr`](../cli/abr/) and `gateway.opt_in_seconds.hls` for [`qumo hls`](../cli/hls/), in whole seconds, added up over its broadcasts (two broadcasts opted into ABR for a minute each are 120). It is the time each announcement of the session beneath `/.abr/` or `/.hls/` lasted on this relay, from when the relay received it until it ended, the session closed or a revoked ticket ended it, so a control plane can bill an opt-in by the time it was used. Only an opt-in the session's grant covers reaches the relay, so an opt-in a key doesn't hold is never counted. An egress the session opted nothing into has no metric. A broadcast opted in through [RTMP or RTSP ingest](../cli/rtmp/#opting-streams-into-hls-and-abr) counts on the ingest's session, under the ingest's own key; the ingest reports the stream itself under its publisher's key ([rtmp → usage](../cli/rtmp/#usage)), so a control plane bills the publisher's key and leaves the ingest's out, as it does any key of its own. **Every other session is a viewer, and viewers are reported together.** Viewers outnumber publishers by orders of magnitude, and a record per viewer would make the receiver's load grow with the audience. Their bytes are added up per key into one running total since the relay started, sent as a `viewers` record naming the key and the relay's `run`. The `run` is new each time the relay starts, so a restart begins new totals rather than lowering old ones. There is no open or close for viewers, and a total is sent only when it has changed. diff --git a/internal/ingest/auth.go b/internal/ingest/auth.go index 4e9ab881..db01e205 100644 --- a/internal/ingest/auth.go +++ b/internal/ingest/auth.go @@ -17,12 +17,22 @@ import ( // PublishAuthorizer decides whether a pushed stream may publish the broadcast // at path with credential, a ?jwt= the publisher presented ("" for none), -// and returns when that permission lapses: the stream is cut then. The zero -// time never lapses. An error refuses the stream. +// and returns the permission it grants. An error refuses the stream. // // A stream's opt-ins (see [optIn]) are asked for as well, with path the // opt-in's, such as /.abr/live/cam for /live/cam. -type PublishAuthorizer func(credential string, path moqt.BroadcastPath) (lapses time.Time, err error) +type PublishAuthorizer func(credential string, path moqt.BroadcastPath) (Permission, error) + +// A Permission is what a PublishAuthorizer grants a push. +type Permission struct { + // Lapses is when the permission lapses: the stream is cut then. The zero + // time never lapses. + Lapses time.Time + // KeyID is the kid of the key that signed the credential granting it, + // whose usage the push is (see [RTMPConfig.Meter]), or "" when none was + // checked. + KeyID string +} // splitCredential splits a name such as "cam?jwt=A&jwt=B" into the name and // the credentials its query carries as jwt, in order, none for none. @@ -65,21 +75,21 @@ func checkPushPath(path moqt.BroadcastPath) error { // credentials, in order, and returns the first permission granted. Without // credentials it asks for a publish that presented none. When none is // granted it returns the first refusal. -func authorizeAny(authorize PublishAuthorizer, credentials []string, path moqt.BroadcastPath) (time.Time, error) { +func authorizeAny(authorize PublishAuthorizer, credentials []string, path moqt.BroadcastPath) (Permission, error) { if len(credentials) == 0 { return authorize("", path) } var first error for _, credential := range credentials { - lapses, err := authorize(credential, path) + p, err := authorize(credential, path) if err == nil { - return lapses, nil + return p, nil } if first == nil { first = err } } - return time.Time{}, first + return Permission{}, first } // optIn opts sess's broadcast into each of optIns, the egresses the server @@ -91,15 +101,15 @@ func authorizeAny(authorize PublishAuthorizer, credentials []string, path moqt.B func optIn(sess *Session, authorize PublishAuthorizer, credentials []string, optIns []token.OptIn) { for _, o := range optIns { path := optin.Path(o.Directory(), sess.path) - var lapses time.Time + var granted Permission if authorize != nil { var err error - if lapses, err = authorizeAny(authorize, credentials, path); err != nil { + if granted, err = authorizeAny(authorize, credentials, path); err != nil { slog.Info("ingest opt-in refused", "broadcast_path", sess.path, "opt_in", o, "error", err) continue } } - sess.OptIn(o, lapses) + sess.OptIn(o, granted.Lapses) } } diff --git a/internal/ingest/auth_test.go b/internal/ingest/auth_test.go index 96759acb..e9792556 100644 --- a/internal/ingest/auth_test.go +++ b/internal/ingest/auth_test.go @@ -82,11 +82,11 @@ func TestAuthorizeAny(t *testing.T) { } for name, tt := range tests { t.Run(name, func(t *testing.T) { - fake := &fakePublishAuthorizer{answer: func(ask publishAsk) (time.Time, error) { + fake := &fakePublishAuthorizer{answer: func(ask publishAsk) (Permission, error) { if err := tt.answers[ask.credential]; err != nil { - return time.Time{}, err + return Permission{}, err } - return lapses, nil + return Permission{Lapses: lapses, KeyID: ask.credential}, nil }} got, err := authorizeAny(fake.authorize, tt.credentials, "/live/cam") @@ -101,7 +101,8 @@ func TestAuthorizeAny(t *testing.T) { return } require.NoError(t, err) - assert.Equal(t, lapses, got) + assert.Equal(t, Permission{Lapses: lapses, KeyID: tt.wantAsked[len(tt.wantAsked)-1]}, got, + "the permission of the credential that granted it") }) } } @@ -169,7 +170,8 @@ func TestPublishAuthorizer(t *testing.T) { authorize := publishAuthorizer(tt.verifier, tt.beneath) require.NotNil(t, authorize) - lapses, err := authorize(tt.credential, tt.path) + granted, err := authorize(tt.credential, tt.path) + lapses := granted.Lapses switch { case tt.wantErr != nil: @@ -181,6 +183,7 @@ func TestPublishAuthorizer(t *testing.T) { case tt.wantLapse: require.NoError(t, err) assert.WithinDuration(t, time.Now().Add(time.Minute), lapses, 5*time.Second, "cut when the credential expires") + assert.Equal(t, tt.verifier == v, granted.KeyID == key.ID, "the push is the usage of the key that signed it") case tt.verifier == nil: require.NoError(t, err) assert.True(t, lapses.IsZero(), "auth off never cuts") @@ -219,9 +222,10 @@ func TestPublishAuthorizer_OptIn(t *testing.T) { } t.Run("a key that holds the opt-in", func(t *testing.T) { - lapses, err := publishAuthorizer(paid, "/live/")(sign(paidKey), "/.abr/live/cam") + granted, err := publishAuthorizer(paid, "/live/")(sign(paidKey), "/.abr/live/cam") require.NoError(t, err) - assert.WithinDuration(t, time.Now().Add(time.Minute), lapses, 5*time.Second) + assert.WithinDuration(t, time.Now().Add(time.Minute), granted.Lapses, 5*time.Second) + assert.Equal(t, paidKey.ID, granted.KeyID) }) t.Run("a key that does not", func(t *testing.T) { _, err := publishAuthorizer(unpaid, "/live/")(sign(unpaidKey), "/.abr/live/cam") diff --git a/internal/ingest/cmd.go b/internal/ingest/cmd.go index 87583c9f..8372ea53 100644 --- a/internal/ingest/cmd.go +++ b/internal/ingest/cmd.go @@ -19,6 +19,7 @@ import ( "github.com/qumo-dev/qumo/internal/auth" "github.com/qumo-dev/qumo/internal/cors" "github.com/qumo-dev/qumo/internal/envconfig" + "github.com/qumo-dev/qumo/internal/meter" "github.com/qumo-dev/qumo/internal/optin" "github.com/qumo-dev/qumo/internal/quicconfig" "github.com/qumo-dev/qumo/internal/upstream" @@ -41,7 +42,8 @@ const ( // RTMP_SERVE_ADDR - MoQT listen address for subscribers to dial directly (default: ":4433" without RELAY_URL, none with it) // QUMO_AUTH_KEYS - the key set a publisher's credential (?jwt= on the stream key) is verified against, as the relay reads it (default: none, every publish is accepted) // QUMO_AUTH_KEYS_CACHE - a file a downloaded key set is kept in between runs -// QUMO_RELAY_TOKEN - bearer token sent to a key-set URL +// QUMO_RELAY_TOKEN - bearer token sent to a key-set URL and to QUMO_USAGE_URL +// QUMO_USAGE_URL - where each stream's usage is reported, under the key of the publisher's token: the media it took in and its opt-in time (package usage); needs QUMO_AUTH_KEYS (default: none, no reports) // INGEST_OPT_IN - the egresses (hls, abr) a stream is opted into on its publisher's behalf, comma-separated: with auth on, those a second ?jwt= on the stream key may publish the opt-in of (/.abr/live/cam for /live/cam); with it off, every stream's (default: none) // RELAY_URL, RELAY_SIGNING_KEY, RELAY_PUBLISH, RELAY_CA_FILE, RELAY_TLS_INSECURE // - the relay to publish through (upstream.FromEnv lists them) @@ -51,8 +53,8 @@ const ( // QUIC_* - QUIC transport settings (quicconfig.FromEnv lists them) func RunRTMP(_ []string) error { return runPush("rtmp", "RTMP_INGEST_ADDR", defaultRTMPIngestAddr, "RTMP_SERVE_ADDR", - func(addr string, mux *moqt.TrackMux, authorize PublishAuthorizer, optIns []token.OptIn) pushServer { - return NewRTMPServer(RTMPConfig{Addr: addr, TrackMux: mux, Authorize: authorize, OptIns: optIns}) + func(c pushConfig) pushServer { + return NewRTMPServer(RTMPConfig{Addr: c.addr, TrackMux: c.mux, Authorize: c.authorize, OptIns: c.optIns, Meter: c.meter}) }) } @@ -65,8 +67,8 @@ func RunRTMP(_ []string) error { // on the URL it pushes to. func RunRTSP(_ []string) error { return runPush("rtsp", "RTSP_INGEST_ADDR", defaultRTSPIngestAddr, "RTSP_SERVE_ADDR", - func(addr string, mux *moqt.TrackMux, authorize PublishAuthorizer, optIns []token.OptIn) pushServer { - return NewRTSPServer(RTSPConfig{Addr: addr, TrackMux: mux, Authorize: authorize, OptIns: optIns}) + func(c pushConfig) pushServer { + return NewRTSPServer(RTSPConfig{Addr: c.addr, TrackMux: c.mux, Authorize: c.authorize, OptIns: c.optIns, Meter: c.meter}) }) } @@ -81,10 +83,18 @@ var ( _ pushServer = (*RTSPServer)(nil) ) +// pushConfig is what runPush gives the push server it makes. +type pushConfig struct { + addr string + mux *moqt.TrackMux + authorize PublishAuthorizer + optIns []token.OptIn + meter *meter.Reporter // nil: no usage reports +} + // runPush runs the push ingest server newServer makes, named name, until // SIGINT or SIGTERM, with its broadcasts published as originFromEnv says. -func runPush(name, ingestEnv, defaultIngestAddr, serveEnv string, - newServer func(addr string, mux *moqt.TrackMux, authorize PublishAuthorizer, optIns []token.OptIn) pushServer) error { +func runPush(name, ingestEnv, defaultIngestAddr, serveEnv string, newServer func(pushConfig) pushServer) error { ingestAddr := envconfig.String(ingestEnv, defaultIngestAddr) quicConfig, err := quicconfig.FromEnv(nil) if err != nil { @@ -103,12 +113,35 @@ func runPush(name, ingestEnv, defaultIngestAddr, serveEnv string, if err != nil { return err } + m, err := meterFromEnv(name) + if err != nil { + return err + } + // The meter sends what is left once more when meterCtx ends, after the + // server has shut down and every stream has recorded its close. + meterCtx, stopMeter := context.WithCancel(context.Background()) + meterDone := make(chan struct{}) + go func() { + defer close(meterDone) + if m != nil { + m.Run(meterCtx) + } + }() + defer func() { + stopMeter() + <-meterDone + }() trackMux := moqt.NewTrackMux(0) - srv := newServer(ingestAddr, trackMux, authorize, out.optIns) + srv := newServer(pushConfig{addr: ingestAddr, mux: trackMux, authorize: authorize, optIns: out.optIns, meter: m}) log.Println(" Ingest :", ingestAddr) out.logBanner() log.Println(" Auth :", authName) + usageURL := "none" + if m != nil { + usageURL = upstream.RelayName(os.Getenv("QUMO_USAGE_URL")) + } + log.Println(" Usage :", usageURL) go func() { if err := srv.ListenAndServe(ctx); err != nil && ctx.Err() == nil { @@ -265,6 +298,25 @@ func publishAuthorizerFromEnv(ctx context.Context, name, beneath string) (Publis return publishAuthorizer(v, beneath), v.Source(), nil } +// meterFromEnv returns the meter that reports the usage of the ingest +// command name's streams to QUMO_USAGE_URL, or nil when it is unset. Usage +// is reported under the key of each publisher's token, so it needs a key set +// (QUMO_AUTH_KEYS). +func meterFromEnv(name string) (*meter.Reporter, error) { + usageURL := os.Getenv("QUMO_USAGE_URL") + if usageURL == "" { + return nil, nil + } + if os.Getenv("QUMO_AUTH_KEYS") == "" { + return nil, fmt.Errorf("%s: QUMO_USAGE_URL needs QUMO_AUTH_KEYS: a stream's usage is reported under the key of its publisher's token", name) + } + m, err := meter.New(meter.Config{Name: name, URL: usageURL, Token: os.Getenv("QUMO_RELAY_TOKEN")}) + if err != nil { + return nil, fmt.Errorf("%s: QUMO_USAGE_URL: %w", name, err) + } + return m, nil +} + // errOutsidePublish refuses a push the relay would not take from the ingest // server: its path lies outside RELAY_PUBLISH. var errOutsidePublish = errors.New("the path lies outside RELAY_PUBLISH, beneath which the relay takes this server's broadcasts") @@ -288,33 +340,33 @@ func publishAuthorizer(v *auth.Verifier, beneath string) PublishAuthorizer { if v == nil && beneath == "" { return nil } - return func(credential string, path moqt.BroadcastPath) (time.Time, error) { + return func(credential string, path moqt.BroadcastPath) (Permission, error) { broadcast := path if o, ok := optin.Of(path); ok { b, named := optin.Broadcast(string(path), o.Directory()) if !named { // Cleaned, it could name a broadcast outside the // directory the relay grants its exact path in. - return time.Time{}, fmt.Errorf("%s: %w", path, errOptInNamesNone) + return Permission{}, fmt.Errorf("%s: %w", path, errOptInNamesNone) } broadcast = moqt.BroadcastPath(b) } if beneath != "" && !strings.HasPrefix(string(broadcast), beneath) { - return time.Time{}, fmt.Errorf("%s: %w (%s)", path, errOutsidePublish, beneath) + return Permission{}, fmt.Errorf("%s: %w (%s)", path, errOutsidePublish, beneath) } if v == nil { - return time.Time{}, nil + return Permission{}, nil } if credential == "" { - return time.Time{}, errNoCredential + return Permission{}, errNoCredential } g, err := v.Check(credential, token.ActionPublish, path) if err != nil { - return time.Time{}, err + return Permission{}, err } if !g.Allows(token.ActionPublish, path) { - return time.Time{}, fmt.Errorf("the credential may not publish %s", path) + return Permission{}, fmt.Errorf("the credential may not publish %s", path) } - return g.Expires(), nil + return Permission{Lapses: g.Expires(), KeyID: g.KeyID()}, nil } } diff --git a/internal/ingest/fake_publish_authorizer_test.go b/internal/ingest/fake_publish_authorizer_test.go index 396e8e0f..97ce0d10 100644 --- a/internal/ingest/fake_publish_authorizer_test.go +++ b/internal/ingest/fake_publish_authorizer_test.go @@ -14,13 +14,14 @@ type publishAsk struct { path moqt.BroadcastPath } -// fakePublishAuthorizer answers every publish with lapses and err, or with +// fakePublishAuthorizer answers every publish with lapses and keyID, or err, or with // answer when it is set, and records what it was asked. The servers ask from // their connection goroutines, so the record is guarded. type fakePublishAuthorizer struct { lapses time.Time + keyID string err error - answer func(publishAsk) (time.Time, error) + answer func(publishAsk) (Permission, error) mu sync.Mutex asked []publishAsk @@ -28,7 +29,7 @@ type fakePublishAuthorizer struct { var _ PublishAuthorizer = (*fakePublishAuthorizer)(nil).authorize -func (f *fakePublishAuthorizer) authorize(credential string, path moqt.BroadcastPath) (time.Time, error) { +func (f *fakePublishAuthorizer) authorize(credential string, path moqt.BroadcastPath) (Permission, error) { f.mu.Lock() defer f.mu.Unlock() ask := publishAsk{credential: credential, path: path} @@ -36,7 +37,10 @@ func (f *fakePublishAuthorizer) authorize(credential string, path moqt.Broadcast if f.answer != nil { return f.answer(ask) } - return f.lapses, f.err + if f.err != nil { + return Permission{}, f.err + } + return Permission{Lapses: f.lapses, KeyID: f.keyID}, nil } // questions returns what the fake was asked so far. diff --git a/internal/ingest/rtmp.go b/internal/ingest/rtmp.go index b8f9a1db..d3e8d3c7 100644 --- a/internal/ingest/rtmp.go +++ b/internal/ingest/rtmp.go @@ -6,11 +6,12 @@ import ( "log/slog" "net" "sync" - "time" "github.com/qumo-dev/gomoqt/moqt" + "github.com/qumo-dev/qumo/internal/meter" "github.com/qumo-dev/qumo/internal/rtmp" "github.com/qumo-dev/qumo/token" + "github.com/qumo-dev/qumo/usage" ) // RTMPConfig holds configuration for an [RTMPServer]. @@ -32,6 +33,13 @@ type RTMPConfig struct { // is asked for the opt-in's path, such as /.abr/live/cam): a publisher // presents a second ?jwt= for it. Nil opts none in. OptIns []token.OptIn + + // Meter, when set, reports each stream's usage under the key of the + // credential Authorize granted it with (Permission.KeyID): an open, + // the media it took in and its opt-in time every 30 s, and a close + // (usage.IngestMediaBytes, usage.OptInSeconds). A stream no key granted, + // with auth off, is not reported. + Meter *meter.Reporter } // RTMPServer accepts RTMP publish connections and bridges each stream to @@ -153,7 +161,7 @@ func (s *RTMPServer) handleConn(ctx context.Context, conn *rtmp.Conn) { var ( path moqt.BroadcastPath credentials []string - lapses time.Time + granted Permission ) mr, err := conn.AcceptStreamFunc(func(app, streamKey string) error { path, credentials = rtmpTarget(app, streamKey) @@ -164,7 +172,7 @@ func (s *RTMPServer) handleConn(ctx context.Context, conn *rtmp.Conn) { return nil } var err error - lapses, err = authorizeAny(s.config.Authorize, credentials, path) + granted, err = authorizeAny(s.config.Authorize, credentials, path) return err }) if err != nil { @@ -178,7 +186,7 @@ func (s *RTMPServer) handleConn(ctx context.Context, conn *rtmp.Conn) { } defer mr.Close() - ctx, cancel := untilLapse(ctx, lapses) + ctx, cancel := untilLapse(ctx, granted.Lapses) defer cancel() // A read blocked on the connection doesn't see ctx end; closing the // connection ends it. @@ -198,6 +206,7 @@ func (s *RTMPServer) handleConn(ctx context.Context, conn *rtmp.Conn) { } defer sess.Close() optIn(sess, s.config.Authorize, credentials, s.config.OptIns) + defer meterStream(s.config.Meter, granted.KeyID, sess)(usage.ReasonClosed) slog.Info("RTMP ingest started", "remote", conn.RemoteAddr(), diff --git a/internal/ingest/rtmp_server_test.go b/internal/ingest/rtmp_server_test.go index 398f542f..48675170 100644 --- a/internal/ingest/rtmp_server_test.go +++ b/internal/ingest/rtmp_server_test.go @@ -23,6 +23,13 @@ func startRTMPServer(t *testing.T, authorize PublishAuthorizer, optIns ...token. t.Helper() mux := moqt.NewTrackMux(0) server := NewRTMPServer(RTMPConfig{Addr: "127.0.0.1:0", TrackMux: mux, Authorize: authorize, OptIns: optIns}) + return mux, serveRTMP(t, server) +} + +// serveRTMP serves server until test cleanup, and returns its address once +// it is listening. +func serveRTMP(t *testing.T, server *RTMPServer) net.Addr { + t.Helper() ctx, cancel := context.WithCancel(context.Background()) errCh := make(chan error, 1) go func() { errCh <- server.ListenAndServe(ctx) }() @@ -36,7 +43,7 @@ func startRTMPServer(t *testing.T, authorize PublishAuthorizer, optIns ...token. addr = server.Addr() return addr != nil }, 2*time.Second, 10*time.Millisecond, "RTMP server never started listening") - return mux, addr + return addr } // publishRTMP opens a publish of app/streamKey on the server at addr. The @@ -121,11 +128,11 @@ func TestRTMPServer_OptIns(t *testing.T) { t.Run(name, func(t *testing.T) { var authorize PublishAuthorizer if tt.authorize { - fake := &fakePublishAuthorizer{answer: func(ask publishAsk) (time.Time, error) { + fake := &fakePublishAuthorizer{answer: func(ask publishAsk) (Permission, error) { if grants[ask] { - return time.Time{}, nil + return Permission{}, nil } - return time.Time{}, errNoGrant + return Permission{}, errNoGrant }} authorize = fake.authorize } diff --git a/internal/ingest/rtsp.go b/internal/ingest/rtsp.go index 2cdeab21..9278b482 100644 --- a/internal/ingest/rtsp.go +++ b/internal/ingest/rtsp.go @@ -13,8 +13,10 @@ import ( "time" "github.com/qumo-dev/gomoqt/moqt" + "github.com/qumo-dev/qumo/internal/meter" "github.com/qumo-dev/qumo/internal/rtsp" "github.com/qumo-dev/qumo/token" + "github.com/qumo-dev/qumo/usage" ) // RTSPConfig holds configuration for an [RTSPServer]. @@ -34,6 +36,10 @@ type RTSPConfig struct { // OptIns are the egresses each stream is opted into, as // [RTMPConfig.OptIns] are. OptIns []token.OptIn + + // Meter, when set, reports each stream's usage under the key that + // granted it, as [RTMPConfig.Meter] does. + Meter *meter.Reporter } // RTSPServer accepts RTSP push connections and bridges each stream to @@ -130,10 +136,13 @@ func (s *RTSPServer) handleConn(ctx context.Context, conn *rtsp.Conn) { sess *Session tracks = make(map[uint8]*rtspTrack) // channel -> track sdp *rtsp.SDP + // endUsage ends the usage report of sess. + endUsage = func(string) {} ) defer func() { if sess != nil { + endUsage(usage.ReasonClosed) sess.Close() } }() @@ -198,8 +207,9 @@ func (s *RTSPServer) handleConn(ctx context.Context, conn *rtsp.Conn) { break } credentials := req.URL.Query()["jwt"] + var granted Permission if s.config.Authorize != nil { - lapses, err := authorizeAny(s.config.Authorize, credentials, path) + granted, err = authorizeAny(s.config.Authorize, credentials, path) if err != nil { slog.Warn("RTSP publish refused", "remote", conn.RemoteAddr(), @@ -213,8 +223,8 @@ func (s *RTSPServer) handleConn(ctx context.Context, conn *rtsp.Conn) { cut.Stop() cut = nil } - if !lapses.IsZero() { - cut = time.AfterFunc(time.Until(lapses), func() { + if !granted.Lapses.IsZero() { + cut = time.AfterFunc(time.Until(granted.Lapses), func() { slog.Info("RTSP ingest cut: the publisher's credential lapsed", "remote", conn.RemoteAddr(), "broadcast_path", path, @@ -228,15 +238,18 @@ func (s *RTSPServer) handleConn(ctx context.Context, conn *rtsp.Conn) { // one leaves the connection as it was. sdp = rtsp.ParseSDP(string(body)) if sess != nil { + endUsage(usage.ReasonClosed) sess.Close() } sess, err = NewSession(s.config.TrackMux, path) if err != nil { slog.Error("failed to create ingest session", "error", err) resp.StatusCode = rtsp.StatusInternalServerError + endUsage = func(string) {} break } optIn(sess, s.config.Authorize, credentials, s.config.OptIns) + endUsage = meterStream(s.config.Meter, granted.KeyID, sess) case rtsp.MethodSetup: if sdp == nil || sess == nil { diff --git a/internal/ingest/rtsp_test.go b/internal/ingest/rtsp_test.go index da1f4499..9ce54e42 100644 --- a/internal/ingest/rtsp_test.go +++ b/internal/ingest/rtsp_test.go @@ -241,11 +241,11 @@ func TestRTSPServer_Authorize(t *testing.T) { // second ?jwt= on its URL that may publish the opt-in, asked for every // credential in order. func TestRTSPServer_OptIns(t *testing.T) { - authorize := &fakePublishAuthorizer{answer: func(ask publishAsk) (time.Time, error) { + authorize := &fakePublishAuthorizer{answer: func(ask publishAsk) (Permission, error) { if ask.path == "/.abr/live/cam" && ask.credential != "abr" { - return time.Time{}, errors.New("no grant") + return Permission{}, errors.New("no grant") } - return time.Time{}, nil + return Permission{}, nil }} server, addr := startRTSPServer(t, authorize.authorize, token.OptInABR) rc := dialRTSP(t, addr) diff --git a/internal/ingest/session.go b/internal/ingest/session.go index 670724cc..b742a3d0 100644 --- a/internal/ingest/session.go +++ b/internal/ingest/session.go @@ -25,6 +25,7 @@ import ( "context" "fmt" "log/slog" + "sync/atomic" "time" "github.com/qumo-dev/gomoqt/moqt" @@ -48,6 +49,10 @@ type Session struct { // ctx ends when the session closes, and with it every opt-in. ctx context.Context cancel context.CancelFunc + // mediaBytes counts the media pushed: the payload of every frame. + mediaBytes atomic.Uint64 + // optIns counts how long the session's opt-ins last. + optIns *optin.Clock } // NewSession creates a new ingest session that announces path on trackMux. @@ -79,6 +84,7 @@ func NewSession(trackMux *moqt.TrackMux, path moqt.BroadcastPath) (*Session, err endAnn: endAnn, ctx: ctx, cancel: cancel, + optIns: optin.NewClock(time.Now), }, nil } @@ -100,9 +106,17 @@ func (s *Session) OptIn(o token.OptIn, lapses time.Time) { slog.Info("ingest opt-in withdrawn", "broadcast_path", s.path, "opt_in", o) }) s.trackMux.Announce(ann, optin.Handler(s.path)) + s.optIns.Track(ann) slog.Info("ingest opted in", "broadcast_path", s.path, "opt_in", o, "opt_in_path", path) } +// Usage returns what the session has taken in so far: the bytes of media +// pushed into it, and how long it has kept its broadcast opted into each +// egress (nil for none). +func (s *Session) Usage() (mediaBytes uint64, optIns map[token.OptIn]time.Duration) { + return s.mediaBytes.Load(), s.optIns.Totals() +} + // RegisterVideo registers (or replaces) the video track in the MSF catalog // using codec configuration extracted from a sequence header. Subscribers // can discover the track via the catalog and receive video frames pushed @@ -125,6 +139,7 @@ func (s *Session) RegisterAudio(cfg *AACConfig) error { // // timestampUS is the presentation timestamp in microseconds. func (s *Session) PushVideo(timestampUS int64, data []byte, isKeyframe bool) { + s.mediaBytes.Add(uint64(len(data))) f := moqt.NewFrame(mediaFrameSize(timestampUS, len(data))) writeMediaFrame(f, timestampUS, data) s.handler.video.push(f, isKeyframe, timestampUS) @@ -135,6 +150,7 @@ func (s *Session) PushVideo(timestampUS int64, data []byte, isKeyframe bool) { // // timestampUS is the presentation timestamp in microseconds. func (s *Session) PushAudio(timestampUS int64, data []byte) { + s.mediaBytes.Add(uint64(len(data))) f := moqt.NewFrame(mediaFrameSize(timestampUS, len(data))) writeMediaFrame(f, timestampUS, data) s.handler.audio.push(f) @@ -156,6 +172,7 @@ func (s *Session) PushAudioFrames(timestampsUS []int64, frames [][]byte) { } built := make([]*moqt.Frame, len(frames)) for i, data := range frames { + s.mediaBytes.Add(uint64(len(data))) pts := timestampsUS[i] f := moqt.NewFrame(mediaFrameSize(pts, len(data))) writeMediaFrame(f, pts, data) diff --git a/internal/ingest/session_test.go b/internal/ingest/session_test.go index ba4b3a3b..13791c0a 100644 --- a/internal/ingest/session_test.go +++ b/internal/ingest/session_test.go @@ -57,3 +57,23 @@ func TestSession_OptIn(t *testing.T) { assert.Eventually(t, func() bool { return !published(mux, "/.hls/live/cam") }, 3*time.Second, 20*time.Millisecond, "withdrawn with the session") } + +// TestSession_Usage verifies a session counts the media pushed into it, the +// payload of every frame, and how long its opt-ins last. +func TestSession_Usage(t *testing.T) { + sess, err := NewSession(moqt.NewTrackMux(0), "/live/cam") + require.NoError(t, err) + defer sess.Close() + + sess.PushVideo(0, make([]byte, 1000), true) + sess.PushAudio(0, make([]byte, 100)) + sess.PushAudioFrames([]int64{0, 21}, [][]byte{make([]byte, 10), make([]byte, 20)}) + media, optIns := sess.Usage() + assert.Equal(t, uint64(1130), media) + assert.Nil(t, optIns, "opted into nothing") + + sess.OptIn(token.OptInABR, time.Time{}) + time.Sleep(20 * time.Millisecond) + _, optIns = sess.Usage() + assert.Greater(t, optIns[token.OptInABR], time.Duration(0), "an opt-in counts while it lasts") +} diff --git a/internal/ingest/usage.go b/internal/ingest/usage.go new file mode 100644 index 00000000..adc9e498 --- /dev/null +++ b/internal/ingest/usage.go @@ -0,0 +1,60 @@ +package ingest + +import ( + "context" + "crypto/rand" + "math" + "time" + + "github.com/qumo-dev/qumo/internal/meter" + "github.com/qumo-dev/qumo/usage" +) + +// usageInterval is how often a pushed stream's usage is reported while it +// runs, as a relay revalidates a session. +const usageInterval = 30 * time.Second + +// meterStream reports the usage of sess, a stream pushed under the key kid, +// to m until the returned function is called with why it ended: an open now, +// its cumulative usage every usageInterval, and a close with its final +// totals. With no meter or no key (auth off) it reports nothing. +func meterStream(m *meter.Reporter, kid string, sess *Session) (end func(reason string)) { + if m == nil || kid == "" { + return func(string) {} + } + id := rand.Text() + m.Open(id, kid) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + ticker := time.NewTicker(usageInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + m.Report(id, kid, streamMetrics(sess)) + } + } + }() + return func(reason string) { + cancel() + <-done + m.Close(id, kid, streamMetrics(sess), reason) + } +} + +// streamMetrics is a pushed stream's usage: the media it took in, and how +// long it kept its broadcast opted into each egress, in whole seconds. +func streamMetrics(sess *Session) map[string]int64 { + mediaBytes, optIns := sess.Usage() + m := map[string]int64{usage.IngestMediaBytes: int64(min(mediaBytes, math.MaxInt64))} + for o, d := range optIns { + if secs := int64(d / time.Second); secs > 0 { + m[usage.OptInSeconds(string(o))] = secs + } + } + return m +} diff --git a/internal/ingest/usage_test.go b/internal/ingest/usage_test.go new file mode 100644 index 00000000..249c5378 --- /dev/null +++ b/internal/ingest/usage_test.go @@ -0,0 +1,123 @@ +package ingest + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/qumo-dev/gomoqt/moqt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/qumo-dev/qumo/internal/meter" + "github.com/qumo-dev/qumo/token" + "github.com/qumo-dev/qumo/usage" +) + +// usageSink records the usage records POSTed to it. +type usageSink struct { + mu sync.Mutex + records []usage.Record +} + +func (s *usageSink) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var batch []usage.Record + if err := json.NewDecoder(r.Body).Decode(&batch); err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.records = append(s.records, batch...) +} + +func (s *usageSink) all() []usage.Record { + s.mu.Lock() + defer s.mu.Unlock() + return append([]usage.Record(nil), s.records...) +} + +// newTestMeter returns a meter sending to a fresh sink. +func newTestMeter(t *testing.T) (*meter.Reporter, *usageSink) { + t.Helper() + sink := &usageSink{} + srv := httptest.NewServer(sink) + t.Cleanup(srv.Close) + m, err := meter.New(meter.Config{Name: "test", URL: srv.URL}) + require.NoError(t, err) + return m, sink +} + +// TestMeterStream verifies a pushed stream is reported under its key: an +// open, and a close carrying the media it took in and its opt-in time. +func TestMeterStream(t *testing.T) { + m, sink := newTestMeter(t) + sess, err := NewSession(moqt.NewTrackMux(0), "/live/cam") + require.NoError(t, err) + defer sess.Close() + + end := meterStream(m, "k1", sess) + sess.PushVideo(0, make([]byte, 500), true) + sess.OptIn(token.OptInABR, time.Time{}) + time.Sleep(1100 * time.Millisecond) + end(usage.ReasonClosed) + require.NoError(t, m.Flush(t.Context())) + + records := sink.all() + require.Len(t, records, 2) + assert.Equal(t, usage.SessionOpen, records[0].Type) + closed := records[1] + assert.Equal(t, usage.SessionClose, closed.Type) + assert.Equal(t, records[0].SessionID, closed.SessionID) + assert.Equal(t, "k1", closed.KID) + assert.Equal(t, usage.ReasonClosed, closed.Reason) + assert.Equal(t, int64(500), closed.Metrics[usage.IngestMediaBytes]) + assert.Equal(t, int64(1), closed.Metrics[usage.OptInSeconds("abr")]) + assert.NotContains(t, closed.Metrics, usage.OptInSeconds("hls")) + for _, rec := range records { + assert.NoError(t, rec.Validate()) + } +} + +// TestMeterStream_Unreported verifies a stream with no meter, or no key +// (auth off), is not reported. +func TestMeterStream_Unreported(t *testing.T) { + m, sink := newTestMeter(t) + sess, err := NewSession(moqt.NewTrackMux(0), "/live/cam") + require.NoError(t, err) + defer sess.Close() + + meterStream(nil, "k1", sess)(usage.ReasonClosed) + meterStream(m, "", sess)(usage.ReasonClosed) + require.NoError(t, m.Flush(t.Context())) + + assert.Empty(t, sink.all()) +} + +// TestRTMPServer_Meter verifies the RTMP server reports a stream under the +// key that granted it, and a refused one not at all. +func TestRTMPServer_Meter(t *testing.T) { + m, sink := newTestMeter(t) + authorize := &fakePublishAuthorizer{answer: func(ask publishAsk) (Permission, error) { + if ask.credential != "tok" { + return Permission{}, errNoCredential + } + return Permission{KeyID: "tenant-key"}, nil + }} + mux := moqt.NewTrackMux(0) + server := NewRTMPServer(RTMPConfig{Addr: "127.0.0.1:0", TrackMux: mux, Authorize: authorize.authorize, Meter: m}) + addr := serveRTMP(t, server) + + publishRTMP(t, addr, "live", "refused?jwt=nope") + publishRTMP(t, addr, "live", "cam?jwt=tok") + require.Eventually(t, func() bool { return published(mux, "/live/cam") }, 3*time.Second, 10*time.Millisecond) + require.NoError(t, m.Flush(t.Context())) + + records := sink.all() + require.Len(t, records, 1, "an open for the stream granted; none for the refused one") + assert.Equal(t, usage.SessionOpen, records[0].Type) + assert.Equal(t, "tenant-key", records[0].KID) +} diff --git a/internal/integration/abr_auth_test.go b/internal/integration/abr_auth_test.go index 7f96ed81..06b94a34 100644 --- a/internal/integration/abr_auth_test.go +++ b/internal/integration/abr_auth_test.go @@ -139,6 +139,21 @@ func runPaidOptIn(t *testing.T, optIns []token.OptIn, paid bool) { "the transcoder, on its own credential, did not publish the ABR broadcast") } +// verifyingAuthorizer checks each push against verifier's key set as the +// relay does, as qumo rtmp's own authorizer with QUMO_AUTH_KEYS does. +func verifyingAuthorizer(verifier *auth.Verifier) ingest.PublishAuthorizer { + return func(credential string, path moqt.BroadcastPath) (ingest.Permission, error) { + g, err := verifier.Check(credential, token.ActionPublish, path) + if err != nil { + return ingest.Permission{}, err + } + if !g.Allows(token.ActionPublish, path) { + return ingest.Permission{}, fmt.Errorf("may not publish %s", path) + } + return ingest.Permission{Lapses: g.Expires(), KeyID: g.KeyID()}, nil + } +} + // TestABR_IngestOptsIn runs the opt-in through RTMP ingest, which opts a // stream in on its publisher's behalf (INGEST_OPT_IN): the publisher puts a // second token, for the opt-in, on its stream key, and the ingest announces @@ -193,16 +208,7 @@ func runIngestOptIn(t *testing.T, optIns []token.OptIn, paid bool) { // The ingest: it checks each publish against the key set as the relay // does, and publishes through the relay with the operator's key. - authorize := func(credential string, path moqt.BroadcastPath) (time.Time, error) { - g, err := verifier.Check(credential, token.ActionPublish, path) - if err != nil { - return time.Time{}, err - } - if !g.Allows(token.ActionPublish, path) { - return time.Time{}, fmt.Errorf("may not publish %s", path) - } - return g.Expires(), nil - } + authorize := verifyingAuthorizer(verifier) ingestMux := moqt.NewTrackMux(0) rtmpLn, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) diff --git a/internal/integration/optin_usage_test.go b/internal/integration/optin_usage_test.go index ce5f3571..af42dbc3 100644 --- a/internal/integration/optin_usage_test.go +++ b/internal/integration/optin_usage_test.go @@ -5,6 +5,8 @@ package integration import ( "context" "encoding/json" + "fmt" + "net" "net/http" "net/http/httptest" "os" @@ -18,6 +20,9 @@ import ( "github.com/stretchr/testify/require" "github.com/qumo-dev/qumo/internal/auth" + "github.com/qumo-dev/qumo/internal/ffpub" + "github.com/qumo-dev/qumo/internal/ingest" + "github.com/qumo-dev/qumo/internal/meter" "github.com/qumo-dev/qumo/internal/optin" "github.com/qumo-dev/qumo/internal/relay" "github.com/qumo-dev/qumo/token" @@ -133,3 +138,88 @@ func TestRelay_ReportsOptInSeconds(t *testing.T) { assert.LessOrEqual(t, secs, int64(5), "the time the opt-in lasted, not longer") assert.NoError(t, closed.Validate()) } + +// TestIngest_ReportsUsage pushes a stream over RTMP with two tokens, for the +// broadcast and for its ABR opt-in, to an ingest that checks them against a +// key set and reports usage. The stream's close record is under the tenant's +// key and carries the media the ingest took in and how long the opt-in +// lasted, so a control plane bills a stream pushed over RTMP as it bills one +// published over MoQ. +func TestIngest_ReportsUsage(t *testing.T) { + if !ffpub.Available() { + t.Skip("ffmpeg not on PATH") + } + tenant, err := token.GenerateKey("/live/") + require.NoError(t, err) + tenantPub := tenant.Public() + tenantPub.OptIns = []token.OptIn{token.OptInABR} + verifier := verifierFor(t, tenantPub) + sign := func(g token.Grant) string { + tok, err := token.Sign(tenant, g, time.Minute) + require.NoError(t, err) + return tok + } + sink := &usageSink{} + usageSrv := httptest.NewServer(sink) + t.Cleanup(usageSrv.Close) + m, err := meter.New(meter.Config{Name: "rtmp", URL: usageSrv.URL}) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + mux := moqt.NewTrackMux(0) + rtmpLn, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + rtmpAddr := rtmpLn.Addr().String() + require.NoError(t, rtmpLn.Close()) + go func() { + _ = ingest.NewRTMPServer(ingest.RTMPConfig{ + Addr: rtmpAddr, TrackMux: mux, Authorize: verifyingAuthorizer(verifier), + OptIns: []token.OptIn{token.OptInABR}, Meter: m, + }).ListenAndServe(ctx) + }() + require.Eventually(t, func() bool { + c, err := net.DialTimeout("tcp", rtmpAddr, 200*time.Millisecond) + if err == nil { + _ = c.Close() + } + return err == nil + }, 5*time.Second, 50*time.Millisecond) + + streamKey := fmt.Sprintf("metered?jwt=%s&jwt=%s", + sign(token.Grant{Root: "/live/", Publish: []string{"/metered"}}), + sign(token.Grant{Root: "/.abr/live/", Publish: []string{"/metered"}})) + pub := ffpub.New(ffpub.Config{ + URL: fmt.Sprintf("rtmp://%s/live/%s", rtmpAddr, streamKey), + GOP: 30, Width: 320, Height: 240, Framerate: 30, + }) + pubCtx, stopPub := context.WithCancel(ctx) + require.NoError(t, pub.Start(pubCtx)) + optInPath := optin.Path(token.ABRDirectory, "/live/metered") + require.Eventually(t, func() bool { + ann, _ := mux.TrackHandler(optInPath) + return ann != nil + }, 10*time.Second, 50*time.Millisecond, "the ingest did not opt the stream in") + + // Opted in for a little over two seconds, then the publisher stops. + time.Sleep(2200 * time.Millisecond) + stopPub() + _ = pub.Wait() + var closed *usage.Record + require.Eventually(t, func() bool { + require.NoError(t, m.Flush(ctx)) + for _, rec := range sink.all() { + if rec.Type == usage.SessionClose { + closed = &rec + } + } + return closed != nil + }, 10*time.Second, 100*time.Millisecond, "no session_close for the stream") + + assert.Equal(t, tenant.ID, closed.KID, "the tenant's key, from its token") + assert.Positive(t, closed.Metrics[usage.IngestMediaBytes], "the media the ingest took in") + secs := closed.Metrics[usage.OptInSeconds("abr")] + assert.GreaterOrEqual(t, secs, int64(2)) + assert.LessOrEqual(t, secs, int64(6)) + assert.NoError(t, closed.Validate()) +} diff --git a/internal/relay/optin_time.go b/internal/optin/clock.go similarity index 54% rename from internal/relay/optin_time.go rename to internal/optin/clock.go index 568b87e2..9d5d4242 100644 --- a/internal/relay/optin_time.go +++ b/internal/optin/clock.go @@ -1,4 +1,4 @@ -package relay +package optin import ( "sync" @@ -6,21 +6,17 @@ import ( "github.com/qumo-dev/gomoqt/moqt" - "github.com/qumo-dev/qumo/internal/optin" "github.com/qumo-dev/qumo/token" ) -// optInClock adds up how long a session keeps broadcasts opted into each -// egress: how long each announcement of its beneath an opt-in directory -// (/.abr/live/cam) lasts, summed over its announcements, so two broadcasts -// opted into ABR for a minute each are two minutes. The session's usage -// reports carry it (auth.Request.OptIns), for a control plane to bill an -// opt-in by the time it was used. A nil clock counts nothing. -// -// It counts what the relay received from the session: an announcement the -// session's grant does not cover never reaches it, so an opt-in the -// session's key does not hold is never counted. -type optInClock struct { +// A Clock adds up how long a publisher keeps broadcasts opted into each +// egress: how long each of its announcements beneath an opt-in directory +// (/.abr/live/cam) lasts, summed over them, so two broadcasts opted into ABR +// for a minute each are two minutes. A server's usage reports carry it +// (usage.OptInSeconds), for a control plane to bill an opt-in by the time it +// was used. A nil Clock counts nothing. Its methods are safe for concurrent +// use. +type Clock struct { now func() time.Time mu sync.Mutex @@ -35,19 +31,20 @@ type optInTally struct { since time.Time } -func newOptInClock(now func() time.Time) *optInClock { - return &optInClock{now: now, tallies: map[token.OptIn]*optInTally{}} +// NewClock returns a Clock reading the time from now (time.Now). +func NewClock(now func() time.Time) *Clock { + return &Clock{now: now, tallies: map[token.OptIn]*optInTally{}} } -// track counts ann, an announcement the session made, until it ends, when it +// Track counts ann, an announcement the publisher made, until it ends, when it // opts a broadcast into an egress; any other is not counted, nor one that // ended already (moqt runs AfterFunc for it later, on another goroutine, // which would count the time until then). -func (c *optInClock) track(ann *moqt.Announcement) { +func (c *Clock) Track(ann *moqt.Announcement) { if c == nil || !ann.IsActive() { return } - o, ok := optin.Of(ann.BroadcastPath()) + o, ok := Of(ann.BroadcastPath()) if !ok { return } @@ -57,7 +54,7 @@ func (c *optInClock) track(ann *moqt.Announcement) { // change adds delta to o's live announcements, counting the time they were // live until now. -func (c *optInClock) change(o token.OptIn, delta int) { +func (c *Clock) change(o token.OptIn, delta int) { c.mu.Lock() defer c.mu.Unlock() now := c.now() @@ -71,9 +68,9 @@ func (c *optInClock) change(o token.OptIn, delta int) { t.since = now } -// totals returns how long the session has kept broadcasts opted into each +// Totals returns how long the publisher has kept broadcasts opted into each // egress so far, live announcements counted up to now, or nil for none. -func (c *optInClock) totals() map[token.OptIn]time.Duration { +func (c *Clock) Totals() map[token.OptIn]time.Duration { if c == nil { return nil } diff --git a/internal/relay/optin_time_test.go b/internal/optin/clock_test.go similarity index 71% rename from internal/relay/optin_time_test.go rename to internal/optin/clock_test.go index be1cda27..ecf83793 100644 --- a/internal/relay/optin_time_test.go +++ b/internal/optin/clock_test.go @@ -1,4 +1,4 @@ -package relay +package optin import ( "context" @@ -25,26 +25,26 @@ func announce(t *testing.T, path moqt.BroadcastPath) (*moqt.Announcement, func() return ann, end } -func TestOptInClock(t *testing.T) { +func TestClock(t *testing.T) { clock := &fakeClock{t: time.Unix(1_700_000_000, 0)} - c := newOptInClock(clock.now) - assert.Nil(t, c.totals(), "nothing opted in yet") + c := NewClock(clock.now) + assert.Nil(t, c.Totals(), "nothing opted in yet") cam1, endCam1 := announce(t, "/.abr/live/cam1") cam2, endCam2 := announce(t, "/.abr/live/cam2") hls, _ := announce(t, "/.hls/live/cam1") plain, _ := announce(t, "/live/cam1") - c.track(cam1) - c.track(plain) + c.Track(cam1) + c.Track(plain) clock.add(10 * time.Second) - c.track(cam2) - c.track(hls) + c.Track(cam2) + c.Track(hls) clock.add(5 * time.Second) assert.Equal(t, map[token.OptIn]time.Duration{ token.OptInABR: 20 * time.Second, // cam1 15 s, cam2 5 s token.OptInHLS: 5 * time.Second, - }, c.totals(), "live announcements count up to now; a broadcast's own announcement is not an opt-in") + }, c.Totals(), "live announcements count up to now; a broadcast's own announcement is not an opt-in") endCam1() clock.add(10 * time.Second) @@ -54,24 +54,24 @@ func TestOptInClock(t *testing.T) { assert.Equal(t, map[token.OptIn]time.Duration{ token.OptInABR: 30 * time.Second, // cam1 0–15 s, cam2 10–25 s token.OptInHLS: time.Hour + 15*time.Second, - }, c.totals(), "an ended announcement stops counting") + }, c.Totals(), "an ended announcement stops counting") } -func TestOptInClock_EndedAnnouncement(t *testing.T) { +func TestClock_EndedAnnouncement(t *testing.T) { clock := &fakeClock{t: time.Unix(1_700_000_000, 0)} - c := newOptInClock(clock.now) + c := NewClock(clock.now) ann, end := announce(t, "/.abr/live/cam1") end() - c.track(ann) + c.Track(ann) clock.add(time.Minute) - assert.Nil(t, c.totals(), "an announcement that ended before it was tracked counts nothing") + assert.Nil(t, c.Totals(), "an announcement that ended before it was tracked counts nothing") } -func TestOptInClock_Nil(t *testing.T) { - var c *optInClock +func TestClock_Nil(t *testing.T) { + var c *Clock ann, _ := announce(t, "/.abr/live/cam1") - c.track(ann) - assert.Nil(t, c.totals(), "an unchecked session counts nothing") + c.Track(ann) + assert.Nil(t, c.Totals(), "a nil Clock counts nothing") } diff --git a/internal/relay/admit.go b/internal/relay/admit.go index 4030d9f8..36a4b052 100644 --- a/internal/relay/admit.go +++ b/internal/relay/admit.go @@ -15,6 +15,7 @@ import ( "github.com/qumo-dev/gomoqt/moqt" "github.com/qumo-dev/qumo/internal/auth" + "github.com/qumo-dev/qumo/internal/optin" "github.com/qumo-dev/qumo/token" "github.com/qumo-dev/qumo/usage" ) @@ -702,13 +703,13 @@ func (s *Server) admitClient(ctx context.Context, req auth.Request) (g *auth.Gra // ended. ctx is the session's: done by now, // so the report keeps its values but not its cancellation. A failed report is // recorded and otherwise ignored; End bounds its own wait. -func (s *Server) reportEnd(ctx context.Context, req auth.Request, stats moqt.SessionStats, optIns *optInClock, reason string) { +func (s *Server) reportEnd(ctx context.Context, req auth.Request, stats moqt.SessionStats, optIns *optin.Clock, reason string) { if s.End == nil { return } req.Event = auth.EventEnd req.Bytes = sessionBytes(stats) - req.OptIns = optIns.totals() + req.OptIns = optIns.Totals() req.Reason = reason if err := s.End(context.WithoutCancel(ctx), req); err != nil { metricAuthRequests.WithLabelValues(auth.EventEnd, authError).Inc() diff --git a/internal/relay/lease.go b/internal/relay/lease.go index 53d0ae12..c43fb324 100644 --- a/internal/relay/lease.go +++ b/internal/relay/lease.go @@ -13,6 +13,7 @@ import ( "github.com/qumo-dev/gomoqt/moqt" "github.com/qumo-dev/gomoqt/transport" "github.com/qumo-dev/qumo/internal/auth" + "github.com/qumo-dev/qumo/internal/optin" "github.com/qumo-dev/qumo/usage" ) @@ -61,7 +62,7 @@ type lease struct { req auth.Request // optIns is the session's opt-in time, sent with each revalidate; nil // for none. - optIns *optInClock + optIns *optin.Clock mu sync.Mutex timer *time.Timer @@ -77,7 +78,7 @@ type lease struct { // startLease starts the lease of a session admitted by req, to be revalidated // every cadence, reporting the opt-in time optIns counts (nil for none). It returns nil for a zero cadence, which never revalidates; // otherwise the caller stops it when the session ends first. -func startLease(ctx context.Context, sess leasedSession, authorize authorizeFunc, req auth.Request, cadence time.Duration, optIns *optInClock) *lease { +func startLease(ctx context.Context, sess leasedSession, authorize authorizeFunc, req auth.Request, cadence time.Duration, optIns *optin.Clock) *lease { if cadence <= 0 { return nil } @@ -171,7 +172,7 @@ func (l *lease) revalidate() (*auth.Grant, error) { req := l.req req.Event = auth.EventRevalidate req.Bytes = sessionBytes(l.sess.Stats()) - req.OptIns = l.optIns.totals() + req.OptIns = l.optIns.Totals() g, err := l.authorize(l.ctx, req) metricAuthRequests.WithLabelValues(auth.EventRevalidate, authOutcome(g, err)).Inc() return g, err diff --git a/internal/relay/server.go b/internal/relay/server.go index 0181cee8..e634c520 100644 --- a/internal/relay/server.go +++ b/internal/relay/server.go @@ -14,6 +14,7 @@ import ( "github.com/qumo-dev/gomoqt/moqt" "github.com/qumo-dev/qumo/internal/auth" "github.com/qumo-dev/qumo/internal/cors" + "github.com/qumo-dev/qumo/internal/optin" ) type Server struct { @@ -811,9 +812,9 @@ func (s *Server) serveSession(sess *moqt.Session) { var l *lease // optIns counts the opt-in time of a checked session, whose usage is // reported; nil counts none. - var optIns *optInClock + var optIns *optin.Clock if checked { - optIns = newOptInClock(time.Now) + optIns = optin.NewClock(time.Now) a.served.Store(true) // Registered first so that it runs last: after the close below, // with the session's final byte totals and close cause. Counted in @@ -878,7 +879,7 @@ func (s *Server) serveSession(sess *moqt.Session) { return } - optIns.track(ann) + optIns.Track(ann) slog.Debug("relay: received announcement", "node", s.Config.NodeID, diff --git a/usage/usage.go b/usage/usage.go index d434a585..1137cecb 100644 --- a/usage/usage.go +++ b/usage/usage.go @@ -13,6 +13,11 @@ // opted into each egress (OptInSeconds), so that a control plane can bill an // option such as ABR by the time it was used. // +// An RTMP or RTSP ingest sends the same records for each stream pushed to +// it, under the key of the publisher's token: SessionOpen, Usage and +// SessionClose with the media it took in (IngestMediaBytes) and its opt-in +// time. +// // Byte counts are cumulative and from the relay's point of view (what it // received is ingress), so a receiver can take a resent record without // counting it twice. A relay that restarts begins a new Run, whose viewer @@ -50,9 +55,13 @@ const ( EgressBytes = "gateway.egress_bytes" ) -// OptInSeconds names the metric of how long a publishing session kept -// broadcasts opted into the egress named optIn ("hls", "abr"), in whole -// seconds, added up over its broadcasts: two broadcasts opted into ABR for +// IngestMediaBytes is the media an RTMP or RTSP ingest took in from a +// publisher: the payload of every frame, without the protocol around it. +const IngestMediaBytes = "ingest.media_bytes" + +// OptInSeconds names the metric of how long a publishing session, on a relay +// or an ingest, kept broadcasts opted into the egress named optIn ("hls", +// "abr"), in whole seconds, added up over its broadcasts: two broadcasts opted into ABR for // a minute each are 120. A session that opted none in carries none. // // OptInSeconds("abr") is "gateway.opt_in_seconds.abr" From 7b067c88b5e6fb21e2015a305fea9452d4a0fd7d Mon Sep 17 00:00:00 2001 From: daichiDeskTop Date: Wed, 18 Nov 2026 21:26:46 +0900 Subject: [PATCH 2/2] fix(ingest): wait for the stream's open in the RTMP meter test, and share the whole-seconds rule TestRTMPServer_Meter flushed the meter once the broadcast was announced, but the stream's usage report opens after that, so the sink often got nothing. It now flushes until the open arrives. TestMeterStream runs its stream in a synctest bubble instead of sleeping for real time. The rule turning opt-in time into whole-second metrics was written in both auth and ingest; optin.AddSeconds is now the one copy. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KPWMJNAzsnsiqRPREa2JqY --- internal/auth/usage.go | 7 ++---- internal/ingest/usage.go | 7 ++---- internal/ingest/usage_test.go | 42 +++++++++++++++++++++++------------ internal/optin/clock.go | 12 ++++++++++ 4 files changed, 44 insertions(+), 24 deletions(-) diff --git a/internal/auth/usage.go b/internal/auth/usage.go index 7c54638b..97a1a61c 100644 --- a/internal/auth/usage.go +++ b/internal/auth/usage.go @@ -7,6 +7,7 @@ import ( "time" "github.com/qumo-dev/qumo/internal/meter" + "github.com/qumo-dev/qumo/internal/optin" "github.com/qumo-dev/qumo/internal/reqsign" "github.com/qumo-dev/qumo/token" "github.com/qumo-dev/qumo/usage" @@ -171,11 +172,7 @@ func usageMetrics(b Bytes) map[string]int64 { // long it kept broadcasts opted into each egress, in whole seconds. func publisherMetrics(b Bytes, optIns map[token.OptIn]time.Duration) map[string]int64 { m := usageMetrics(b) - for o, d := range optIns { - if secs := int64(d / time.Second); secs > 0 { - m[usage.OptInSeconds(string(o))] = secs - } - } + optin.AddSeconds(m, optIns) return m } diff --git a/internal/ingest/usage.go b/internal/ingest/usage.go index adc9e498..686fc638 100644 --- a/internal/ingest/usage.go +++ b/internal/ingest/usage.go @@ -7,6 +7,7 @@ import ( "time" "github.com/qumo-dev/qumo/internal/meter" + "github.com/qumo-dev/qumo/internal/optin" "github.com/qumo-dev/qumo/usage" ) @@ -51,10 +52,6 @@ func meterStream(m *meter.Reporter, kid string, sess *Session) (end func(reason func streamMetrics(sess *Session) map[string]int64 { mediaBytes, optIns := sess.Usage() m := map[string]int64{usage.IngestMediaBytes: int64(min(mediaBytes, math.MaxInt64))} - for o, d := range optIns { - if secs := int64(d / time.Second); secs > 0 { - m[usage.OptInSeconds(string(o))] = secs - } - } + optin.AddSeconds(m, optIns) return m } diff --git a/internal/ingest/usage_test.go b/internal/ingest/usage_test.go index 249c5378..cc68f069 100644 --- a/internal/ingest/usage_test.go +++ b/internal/ingest/usage_test.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" "sync" "testing" + "testing/synctest" "time" "github.com/qumo-dev/gomoqt/moqt" @@ -55,15 +56,19 @@ func newTestMeter(t *testing.T) (*meter.Reporter, *usageSink) { // open, and a close carrying the media it took in and its opt-in time. func TestMeterStream(t *testing.T) { m, sink := newTestMeter(t) - sess, err := NewSession(moqt.NewTrackMux(0), "/live/cam") - require.NoError(t, err) - defer sess.Close() - - end := meterStream(m, "k1", sess) - sess.PushVideo(0, make([]byte, 500), true) - sess.OptIn(token.OptInABR, time.Time{}) - time.Sleep(1100 * time.Millisecond) - end(usage.ReasonClosed) + // The stream runs in a bubble, so its opt-in lasts exactly as long as + // the sleep; the meter sends over the network outside it. + synctest.Test(t, func(t *testing.T) { + sess, err := NewSession(moqt.NewTrackMux(0), "/live/cam") + require.NoError(t, err) + defer sess.Close() + + end := meterStream(m, "k1", sess) + sess.PushVideo(0, make([]byte, 500), true) + sess.OptIn(token.OptInABR, time.Time{}) + time.Sleep(1500 * time.Millisecond) + end(usage.ReasonClosed) + }) require.NoError(t, m.Flush(t.Context())) records := sink.all() @@ -113,11 +118,20 @@ func TestRTMPServer_Meter(t *testing.T) { publishRTMP(t, addr, "live", "refused?jwt=nope") publishRTMP(t, addr, "live", "cam?jwt=tok") - require.Eventually(t, func() bool { return published(mux, "/live/cam") }, 3*time.Second, 10*time.Millisecond) - require.NoError(t, m.Flush(t.Context())) + // The stream is announced before its usage report opens, so wait for + // the open itself. + var records []usage.Record + require.Eventually(t, func() bool { + if err := m.Flush(t.Context()); err != nil { + return false + } + records = append(records, sink.all()[len(records):]...) + return len(records) > 0 + }, 3*time.Second, 10*time.Millisecond) - records := sink.all() - require.Len(t, records, 1, "an open for the stream granted; none for the refused one") assert.Equal(t, usage.SessionOpen, records[0].Type) - assert.Equal(t, "tenant-key", records[0].KID) + for _, rec := range records { + assert.Equal(t, "tenant-key", rec.KID, "only the stream granted is reported, under the key that granted it") + assert.Equal(t, records[0].SessionID, rec.SessionID) + } } diff --git a/internal/optin/clock.go b/internal/optin/clock.go index 9d5d4242..e957dd56 100644 --- a/internal/optin/clock.go +++ b/internal/optin/clock.go @@ -7,6 +7,7 @@ import ( "github.com/qumo-dev/gomoqt/moqt" "github.com/qumo-dev/qumo/token" + "github.com/qumo-dev/qumo/usage" ) // A Clock adds up how long a publisher keeps broadcasts opted into each @@ -68,6 +69,17 @@ func (c *Clock) change(o token.OptIn, delta int) { t.since = now } +// AddSeconds adds each egress's time in optIns to m, as Totals returns it, +// in whole seconds under usage.OptInSeconds. An egress with less than a +// second is left out, so a session that opted nothing in carries no metric. +func AddSeconds(m map[string]int64, optIns map[token.OptIn]time.Duration) { + for o, d := range optIns { + if secs := int64(d / time.Second); secs > 0 { + m[usage.OptInSeconds(string(o))] = secs + } + } +} + // Totals returns how long the publisher has kept broadcasts opted into each // egress so far, live announcements counted up to now, or nil for none. func (c *Clock) Totals() map[token.OptIn]time.Duration {