diff --git a/CHANGELOG.md b/CHANGELOG.md index fcf19db..1513965 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,12 @@ ### Added +- `Telemetry::gauge(name)` returns a `Gauge` whose `record(value, attrs)` sets + the value exported; the last value is exported again on every collection, so + a series stays present between rare events (for an alert on the time of the + last good run). The trait method has a default no-op implementation, so + existing `Telemetry` implementations keep compiling. + - `AppBuilder::without_end_user_telemetry()` (feature `telemetry`): an end-user install ships no telemetry — no `telemetry` command group, no telemetry doctor checks, no telemetry variables in root help, no first-run diff --git a/docs/telemetry.md b/docs/telemetry.md index 9b6bd59..83d7da0 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -427,8 +427,17 @@ every build, and is a no-op when telemetry is off: ```rust ctx.telemetry().counter("myapp.widgets_created").add(1, &[]); ctx.telemetry().histogram("myapp.render_ms").record(elapsed_ms, &[]); +ctx.telemetry().gauge("myapp.last_sync_unix").record(now_unix, &[]); ``` +A gauge exports the last value recorded, and exports it again on every +collection until a new one is recorded, so its series stays present between +rare events. That is what an alert on "the last good run is too old" needs: a +counter that has not moved since the process started exports nothing, and +reads the same as a process that is not running. `gauge` has a default no-op +implementation, so a `Telemetry` of your own (a test double, say) does not have +to write one. + Instruments emitted this way ride the same pipeline as the built-in metrics. `SpanHandle::set_attr` records only keys declared at the span's callsite — `tracing` fixes a span's fieldset at compile time, so an undeclared key is diff --git a/skill/references/telemetry.md b/skill/references/telemetry.md index 8008015..ee52fe4 100644 --- a/skill/references/telemetry.md +++ b/skill/references/telemetry.md @@ -78,7 +78,9 @@ let builder = builder ``` In a handler: `ctx.telemetry().counter("myapp.x").add(1, &[])` / -`.histogram("myapp.ms").record(v, &[])` — a no-op when telemetry is off. +`.histogram("myapp.ms").record(v, &[])` / `.gauge("myapp.last_ok").record(t, &[])` +— a no-op when telemetry is off. A gauge's last value is exported again on +every collection. `SpanHandle::set_attr` only records keys declared at the span's callsite; `record_error` works. diff --git a/src/lib.rs b/src/lib.rs index b734d4a..6b762f7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -221,7 +221,7 @@ pub mod prelude { PolicyCache, PolicyClient, PolicyClientError, PolicyOutcome, RoamingConfigClient, }; - pub use crate::telemetry::handle::{Counter, Histogram, SpanHandle, Telemetry}; + pub use crate::telemetry::handle::{Counter, Gauge, Histogram, SpanHandle, Telemetry}; pub use crate::telemetry::{NoopTelemetry, TelemetryConfig}; } diff --git a/src/telemetry/handle.rs b/src/telemetry/handle.rs index d4006b6..71c578e 100644 --- a/src/telemetry/handle.rs +++ b/src/telemetry/handle.rs @@ -21,6 +21,18 @@ pub trait Telemetry: Send + Sync { /// Exported over OTLP when a `metrics_enabled` config with an endpoint is /// active; a safe no-op otherwise. fn histogram(&self, name: &str) -> Histogram; + /// Return a gauge handle: the last value recorded is the one exported. + /// + /// The value is re-exported on every collection until a new one is + /// recorded, so a series stays present between rare events (for example + /// the time of the last good backup, which an alert compares with now). + /// Exported over OTLP when a `metrics_enabled` config with an endpoint is + /// active; a safe no-op otherwise, and by default for an implementation + /// that does not override it. + fn gauge(&self, name: &str) -> Gauge { + let _ = name; + Gauge(GaugeInner::Noop) + } /// Open a child span; it closes when the returned [`SpanHandle`] is dropped. fn span(&self, name: &str, attrs: &[KeyValue]) -> SpanHandle; } @@ -61,6 +73,24 @@ impl Histogram { } } +pub struct Gauge(pub(crate) GaugeInner); +pub(crate) enum GaugeInner { + Noop, + #[cfg(feature = "telemetry")] + Live(opentelemetry::metrics::Gauge), +} +impl Gauge { + pub fn record(&self, value: f64, attrs: &[KeyValue]) { + match &self.0 { + GaugeInner::Noop => { + let _ = (value, attrs); + } + #[cfg(feature = "telemetry")] + GaugeInner::Live(g) => g.record(value, attrs), + } + } +} + pub struct SpanHandle(pub(crate) SpanInner); pub(crate) enum SpanInner { Noop, diff --git a/src/telemetry/init.rs b/src/telemetry/init.rs index 9411704..a207685 100644 --- a/src/telemetry/init.rs +++ b/src/telemetry/init.rs @@ -2,7 +2,10 @@ use crate::telemetry::{ config::TelemetryConfig, exporter::RedactingExporter, guard::TelemetryGuard, - handle::{Counter, CounterInner, Histogram, HistogramInner, SpanHandle, SpanInner, Telemetry}, + handle::{ + Counter, CounterInner, Gauge, GaugeInner, Histogram, HistogramInner, SpanHandle, SpanInner, + Telemetry, + }, policy::TelemetryPolicy, redact::METRIC_LABEL_ALLOWLIST, resource::{ @@ -38,6 +41,11 @@ impl Telemetry for LiveTelemetry { self.meter.f64_histogram(name.to_string()).build(), )) } + fn gauge(&self, name: &str) -> Gauge { + Gauge(GaugeInner::Live( + self.meter.f64_gauge(name.to_string()).build(), + )) + } fn span(&self, name: &str, attrs: &[KeyValue]) -> SpanHandle { // `otel.status_code` / `otel.status_description` are special fields that // `tracing-opentelemetry` maps onto the OTel span *status*. They must be diff --git a/src/telemetry/mod.rs b/src/telemetry/mod.rs index 3cc3672..a35ef8c 100644 --- a/src/telemetry/mod.rs +++ b/src/telemetry/mod.rs @@ -94,7 +94,7 @@ pub use guard::TelemetryGuard; // itself, which has a no-op stub for a default build. #[cfg(feature = "telemetry")] pub use guard::{flush_within, flush_within_for_test, FlushOutcome, END_USER_FLUSH_BUDGET}; -pub use handle::{Counter, Histogram, KeyValue, SpanHandle, Telemetry}; +pub use handle::{Counter, Gauge, Histogram, KeyValue, SpanHandle, Telemetry}; pub use noop::NoopTelemetry; pub mod axes; diff --git a/src/telemetry/noop.rs b/src/telemetry/noop.rs index ecb2311..3d5c30c 100644 --- a/src/telemetry/noop.rs +++ b/src/telemetry/noop.rs @@ -1,5 +1,6 @@ use crate::telemetry::handle::{ - Counter, CounterInner, Histogram, HistogramInner, SpanHandle, SpanInner, Telemetry, + Counter, CounterInner, Gauge, GaugeInner, Histogram, HistogramInner, SpanHandle, SpanInner, + Telemetry, }; use opentelemetry::KeyValue; @@ -19,6 +20,9 @@ impl Telemetry for NoopTelemetry { fn histogram(&self, _: &str) -> Histogram { Histogram(HistogramInner::Noop) } + fn gauge(&self, _: &str) -> Gauge { + Gauge(GaugeInner::Noop) + } fn span(&self, _: &str, _: &[KeyValue]) -> SpanHandle { SpanHandle(SpanInner::Noop) } diff --git a/tests/unit/telemetry.rs b/tests/unit/telemetry.rs index dc445ee..b1cb0fa 100644 --- a/tests/unit/telemetry.rs +++ b/tests/unit/telemetry.rs @@ -68,6 +68,7 @@ fn noop_telemetry_is_zero_cost() { t.event("evt", &[]); t.counter("c").add(1, &[]); t.histogram("h").record(1.0, &[]); + t.gauge("g").record(1.0, &[]); // Exercise SpanHandle Noop paths for set_attr and record_error let span = t.span("s", &[]); span.set_attr("key", "val"); @@ -267,6 +268,79 @@ async fn live_telemetry_histogram_record_exercises_live_variant() { drop(guard); } +/// Every value a gauge named `name` reported in the exporter's collections, in +/// order. +fn gauge_values( + exporter: &opentelemetry_sdk::metrics::InMemoryMetricExporter, + name: &str, +) -> Vec { + use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData}; + let mut values = Vec::new(); + for rm in exporter.get_finished_metrics().expect("finished metrics") { + for sm in rm.scope_metrics() { + for m in sm.metrics().filter(|m| m.name() == name) { + if let AggregatedMetrics::F64(MetricData::Gauge(g)) = m.data() { + values.extend(g.data_points().map(|dp| dp.value())); + } + } + } + } + values +} + +#[test] +fn live_telemetry_gauge_keeps_its_last_value_between_collections() { + use opentelemetry::metrics::MeterProvider as _; + use opentelemetry_sdk::metrics::{InMemoryMetricExporter, PeriodicReader, SdkMeterProvider}; + let exporter = InMemoryMetricExporter::default(); + let provider = SdkMeterProvider::builder() + .with_reader(PeriodicReader::builder(exporter.clone()).build()) + .build(); + let handle = cli_framework::telemetry::init::LiveTelemetry::new(provider.meter("test")); + + let gauge = handle.gauge("backup.last_success_unix"); + gauge.record(1_000.0, &[]); + gauge.record(1_760_000_000.0, &[]); + provider.force_flush().expect("first collection"); + assert_eq!( + gauge_values(&exporter, "backup.last_success_unix"), + vec![1_760_000_000.0], + "a gauge exports the last value recorded, not each one" + ); + + // Nothing recorded since: the value is exported again, so the series does + // not go missing between two rare events. + exporter.reset(); + provider.force_flush().expect("second collection"); + assert_eq!( + gauge_values(&exporter, "backup.last_success_unix"), + vec![1_760_000_000.0] + ); + provider.shutdown().expect("shutdown"); +} + +#[test] +fn a_telemetry_impl_without_gauge_gets_a_noop_one() { + struct OnlyEvents; + impl Telemetry for OnlyEvents { + fn event(&self, _: &str, _: &[cli_framework::telemetry::KeyValue]) {} + fn counter(&self, name: &str) -> cli_framework::telemetry::Counter { + NoopTelemetry.counter(name) + } + fn histogram(&self, name: &str) -> cli_framework::telemetry::Histogram { + NoopTelemetry.histogram(name) + } + fn span( + &self, + name: &str, + attrs: &[cli_framework::telemetry::KeyValue], + ) -> cli_framework::telemetry::SpanHandle { + NoopTelemetry.span(name, attrs) + } + } + OnlyEvents.gauge("g").record(1.0, &[]); +} + #[tokio::test] async fn live_telemetry_event_emits() { let _otel = otel_lock();