feat(telemetry): Telemetry::gauge, a last value exported on every collection

CHANGELOG.md

@@ -34,6 +34,12 @@
 
 ### Added
 
+- `Telemetry::gauge(name)` returns a `Gauge` whose `record(value, attrs)` sets
+  the value exported; the last value is exported again on every collection, so
+  a series stays present between rare events (for an alert on the time of the
+  last good run). The trait method has a default no-op implementation, so
+  existing `Telemetry` implementations keep compiling.
+
 - `AppBuilder::without_end_user_telemetry()` (feature `telemetry`): an
   end-user install ships no telemetry — no `telemetry` command group, no
   telemetry doctor checks, no telemetry variables in root help, no first-run

docs/telemetry.md

@@ -427,8 +427,17 @@ every build, and is a no-op when telemetry is off:
 ```rust
 ctx.telemetry().counter("myapp.widgets_created").add(1, &[]);
 ctx.telemetry().histogram("myapp.render_ms").record(elapsed_ms, &[]);
+ctx.telemetry().gauge("myapp.last_sync_unix").record(now_unix, &[]);
 ```
 
+A gauge exports the last value recorded, and exports it again on every
+collection until a new one is recorded, so its series stays present between
+rare events. That is what an alert on "the last good run is too old" needs: a
+counter that has not moved since the process started exports nothing, and
+reads the same as a process that is not running. `gauge` has a default no-op
+implementation, so a `Telemetry` of your own (a test double, say) does not have
+to write one.
+
 Instruments emitted this way ride the same pipeline as the built-in metrics.
 `SpanHandle::set_attr` records only keys declared at the span's callsite —
 `tracing` fixes a span's fieldset at compile time, so an undeclared key is

skill/references/telemetry.md

@@ -78,7 +78,9 @@ let builder = builder
 ```
 
 In a handler: `ctx.telemetry().counter("myapp.x").add(1, &[])` /
-`.histogram("myapp.ms").record(v, &[])` — a no-op when telemetry is off.
+`.histogram("myapp.ms").record(v, &[])` / `.gauge("myapp.last_ok").record(t, &[])`
+— a no-op when telemetry is off. A gauge's last value is exported again on
+every collection.
 `SpanHandle::set_attr` only records keys declared at the span's callsite;
 `record_error` works.
 

src/lib.rs

@@ -221,7 +221,7 @@ pub mod prelude {
         PolicyCache, PolicyClient, PolicyClientError, PolicyOutcome, RoamingConfigClient,
     };
 
-    pub use crate::telemetry::handle::{Counter, Histogram, SpanHandle, Telemetry};
+    pub use crate::telemetry::handle::{Counter, Gauge, Histogram, SpanHandle, Telemetry};
     pub use crate::telemetry::{NoopTelemetry, TelemetryConfig};
 }
 

src/telemetry/handle.rs

@@ -21,6 +21,18 @@ pub trait Telemetry: Send + Sync {
     /// Exported over OTLP when a `metrics_enabled` config with an endpoint is
     /// active; a safe no-op otherwise.
     fn histogram(&self, name: &str) -> Histogram;
+    /// Return a gauge handle: the last value recorded is the one exported.
+    ///
+    /// The value is re-exported on every collection until a new one is
+    /// recorded, so a series stays present between rare events (for example
+    /// the time of the last good backup, which an alert compares with now).
+    /// Exported over OTLP when a `metrics_enabled` config with an endpoint is
+    /// active; a safe no-op otherwise, and by default for an implementation
+    /// that does not override it.
+    fn gauge(&self, name: &str) -> Gauge {
+        let _ = name;
+        Gauge(GaugeInner::Noop)
+    }
     /// Open a child span; it closes when the returned [`SpanHandle`] is dropped.
     fn span(&self, name: &str, attrs: &[KeyValue]) -> SpanHandle;
 }
@@ -61,6 +73,24 @@ impl Histogram {
     }
 }
 
+pub struct Gauge(pub(crate) GaugeInner);
+pub(crate) enum GaugeInner {
+    Noop,
+    #[cfg(feature = "telemetry")]
+    Live(opentelemetry::metrics::Gauge<f64>),
+}
+impl Gauge {
+    pub fn record(&self, value: f64, attrs: &[KeyValue]) {
+        match &self.0 {
+            GaugeInner::Noop => {
+                let _ = (value, attrs);
+            }
+            #[cfg(feature = "telemetry")]
+            GaugeInner::Live(g) => g.record(value, attrs),
+        }
+    }
+}
+
 pub struct SpanHandle(pub(crate) SpanInner);
 pub(crate) enum SpanInner {
     Noop,

src/telemetry/init.rs

@@ -2,7 +2,10 @@ use crate::telemetry::{
     config::TelemetryConfig,
     exporter::RedactingExporter,
     guard::TelemetryGuard,
-    handle::{Counter, CounterInner, Histogram, HistogramInner, SpanHandle, SpanInner, Telemetry},
+    handle::{
+        Counter, CounterInner, Gauge, GaugeInner, Histogram, HistogramInner, SpanHandle, SpanInner,
+        Telemetry,
+    },
     policy::TelemetryPolicy,
     redact::METRIC_LABEL_ALLOWLIST,
     resource::{
@@ -38,6 +41,11 @@ impl Telemetry for LiveTelemetry {
             self.meter.f64_histogram(name.to_string()).build(),
         ))
     }
+    fn gauge(&self, name: &str) -> Gauge {
+        Gauge(GaugeInner::Live(
+            self.meter.f64_gauge(name.to_string()).build(),
+        ))
+    }
     fn span(&self, name: &str, attrs: &[KeyValue]) -> SpanHandle {
         // `otel.status_code` / `otel.status_description` are special fields that
         // `tracing-opentelemetry` maps onto the OTel span *status*. They must be

src/telemetry/mod.rs

@@ -94,7 +94,7 @@ pub use guard::TelemetryGuard;
 // itself, which has a no-op stub for a default build.
 #[cfg(feature = "telemetry")]
 pub use guard::{flush_within, flush_within_for_test, FlushOutcome, END_USER_FLUSH_BUDGET};
-pub use handle::{Counter, Histogram, KeyValue, SpanHandle, Telemetry};
+pub use handle::{Counter, Gauge, Histogram, KeyValue, SpanHandle, Telemetry};
 pub use noop::NoopTelemetry;
 
 pub mod axes;

src/telemetry/noop.rs

@@ -1,5 +1,6 @@
 use crate::telemetry::handle::{
-    Counter, CounterInner, Histogram, HistogramInner, SpanHandle, SpanInner, Telemetry,
+    Counter, CounterInner, Gauge, GaugeInner, Histogram, HistogramInner, SpanHandle, SpanInner,
+    Telemetry,
 };
 use opentelemetry::KeyValue;
 
@@ -19,6 +20,9 @@ impl Telemetry for NoopTelemetry {
     fn histogram(&self, _: &str) -> Histogram {
         Histogram(HistogramInner::Noop)
     }
+    fn gauge(&self, _: &str) -> Gauge {
+        Gauge(GaugeInner::Noop)
+    }
     fn span(&self, _: &str, _: &[KeyValue]) -> SpanHandle {
         SpanHandle(SpanInner::Noop)
     }

tests/unit/telemetry.rs

@@ -68,6 +68,7 @@ fn noop_telemetry_is_zero_cost() {
     t.event("evt", &[]);
     t.counter("c").add(1, &[]);
     t.histogram("h").record(1.0, &[]);
+    t.gauge("g").record(1.0, &[]);
     // Exercise SpanHandle Noop paths for set_attr and record_error
     let span = t.span("s", &[]);
     span.set_attr("key", "val");
@@ -267,6 +268,79 @@ async fn live_telemetry_histogram_record_exercises_live_variant() {
     drop(guard);
 }
 
+/// Every value a gauge named `name` reported in the exporter's collections, in
+/// order.
+fn gauge_values(
+    exporter: &opentelemetry_sdk::metrics::InMemoryMetricExporter,
+    name: &str,
+) -> Vec<f64> {
+    use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData};
+    let mut values = Vec::new();
+    for rm in exporter.get_finished_metrics().expect("finished metrics") {
+        for sm in rm.scope_metrics() {
+            for m in sm.metrics().filter(|m| m.name() == name) {
+                if let AggregatedMetrics::F64(MetricData::Gauge(g)) = m.data() {
+                    values.extend(g.data_points().map(|dp| dp.value()));
+                }
+            }
+        }
+    }
+    values
+}
+
+#[test]
+fn live_telemetry_gauge_keeps_its_last_value_between_collections() {
+    use opentelemetry::metrics::MeterProvider as _;
+    use opentelemetry_sdk::metrics::{InMemoryMetricExporter, PeriodicReader, SdkMeterProvider};
+    let exporter = InMemoryMetricExporter::default();
+    let provider = SdkMeterProvider::builder()
+        .with_reader(PeriodicReader::builder(exporter.clone()).build())
+        .build();
+    let handle = cli_framework::telemetry::init::LiveTelemetry::new(provider.meter("test"));
+
+    let gauge = handle.gauge("backup.last_success_unix");
+    gauge.record(1_000.0, &[]);
+    gauge.record(1_760_000_000.0, &[]);
+    provider.force_flush().expect("first collection");
+    assert_eq!(
+        gauge_values(&exporter, "backup.last_success_unix"),
+        vec![1_760_000_000.0],
+        "a gauge exports the last value recorded, not each one"
+    );
+
+    // Nothing recorded since: the value is exported again, so the series does
+    // not go missing between two rare events.
+    exporter.reset();
+    provider.force_flush().expect("second collection");
+    assert_eq!(
+        gauge_values(&exporter, "backup.last_success_unix"),
+        vec![1_760_000_000.0]
+    );
+    provider.shutdown().expect("shutdown");
+}
+
+#[test]
+fn a_telemetry_impl_without_gauge_gets_a_noop_one() {
+    struct OnlyEvents;
+    impl Telemetry for OnlyEvents {
+        fn event(&self, _: &str, _: &[cli_framework::telemetry::KeyValue]) {}
+        fn counter(&self, name: &str) -> cli_framework::telemetry::Counter {
+            NoopTelemetry.counter(name)
+        }
+        fn histogram(&self, name: &str) -> cli_framework::telemetry::Histogram {
+            NoopTelemetry.histogram(name)
+        }
+        fn span(
+            &self,
+            name: &str,
+            attrs: &[cli_framework::telemetry::KeyValue],
+        ) -> cli_framework::telemetry::SpanHandle {
+            NoopTelemetry.span(name, attrs)
+        }
+    }
+    OnlyEvents.gauge("g").record(1.0, &[]);
+}
+
 #[tokio::test]
 async fn live_telemetry_event_emits() {
     let _otel = otel_lock();