ROX-35938: Add secured cluster cert expiry to expiry metrics tracker

CHANGELOG.md

@@ -16,6 +16,7 @@ Changes should still be described appropriately in JIRA/doc input pages, for inc
 - ROX-34997: The Central CR now supports `spec.central.rolloutStrategy` (`Recreate` or `RollingUpdate`) to configure the central deployment rollout strategy. Default remains `Recreate`.
 
 - ROX-35181: Administrative events are now exposed as configurable custom Prometheus metrics (`rox_central_admin_event_*`), aggregated by Type, Level, Domain, ResourceType, and ResourceName. Requires permission to read Administration resource, globally scoped.
+- ROX-35938: The `rox_central_cert_exp_hours` custom Prometheus metric now also reports secured cluster (Sensor) certificate expiry, as a `SECURED_CLUSTER` component with the cluster name in a new `Name` label.
 - ROX-35545: Added ACL change as a file access operation for runtime policies.
 
 ### Removed Features

central/metrics/custom/expiry/labels.go

@@ -6,10 +6,12 @@ import (
 
 var LazyLabels = tracker.LazyLabelGetters[*finding]{
 	"Component": func(f *finding) string { return f.component },
+	"Name":      func(f *finding) string { return f.name },
 }
 
 type finding struct {
 	component            string
+	name                 string
 	hoursUntilExpiration int
 }
 

central/metrics/custom/expiry/tracker.go

@@ -4,51 +4,73 @@ import (
 	"context"
 	"time"
 
+	clusterDS "github.com/stackrox/rox/central/cluster/datastore"
 	"github.com/stackrox/rox/central/credentialexpiry/service"
 	"github.com/stackrox/rox/central/metrics"
 	"github.com/stackrox/rox/central/metrics/custom/tracker"
 	v1 "github.com/stackrox/rox/generated/api/v1"
+	"github.com/stackrox/rox/generated/storage"
 	"github.com/stackrox/rox/pkg/logging"
 )
 
-func New(s service.Service) *tracker.TrackerBase[*finding] {
-	return tracker.MakeGlobalTrackerBase(
+// securedClusterComponent is a synthetic component, not part of the
+// v1.GetCertExpiry_Component enum: unlike Central/Scanner/Central DB/Scanner
+// V4, there can be any number of secured clusters, each identified by the
+// "Name" label rather than by a fixed enum value.
+const securedClusterComponent = "SECURED_CLUSTER"
+
+func New(s service.Service, clusters clusterDS.DataStore) *tracker.TrackerBase[*finding] {
+	return tracker.MakeTrackerBase(
 		metrics.Expiry,
 		"hours before certificate expires",
 		LazyLabels,
 		func(ctx context.Context, _ tracker.MetricDescriptors) tracker.FindingErrorSequence[*finding] {
-			return track(ctx, s)
+			return track(ctx, s, clusters)
 		},
 	)
 }
 
-func track(ctx context.Context, s service.Service) tracker.FindingErrorSequence[*finding] {
+func track(ctx context.Context, s service.Service, clusters clusterDS.DataStore) tracker.FindingErrorSequence[*finding] {
 	return func(yield func(*finding, error) bool) {
-		if s == nil {
+		if s != nil {
+			for i, component := range v1.GetCertExpiry_Component_name {
+				if v1.GetCertExpiry_Component(i) == v1.GetCertExpiry_UNKNOWN {
+					continue
+				}
+				result, err := s.GetCertExpiry(ctx, &v1.GetCertExpiry_Request{
+					Component: v1.GetCertExpiry_Component(i),
+				})
+				if err != nil {
+					// Ignore particular component errors, as they do not affect
+					// other components metrics.
+					logging.LoggerForModule().Errorw("Failed to get certificate expiry",
+						logging.String("component", component), logging.Err(err))
+					continue
+				}
+				f := &finding{component: component}
+				if result != nil {
+					f.hoursUntilExpiration = int(time.Until(result.GetExpiry().AsTime()).Hours())
+				}
+				if !yield(f, nil) {
+					return
+				}
+			}
+		}
+		if clusters == nil {
 			return
 		}
-		var f finding
-		for i, component := range v1.GetCertExpiry_Component_name {
-			if v1.GetCertExpiry_Component(i) == v1.GetCertExpiry_UNKNOWN {
-				continue
+		collector := tracker.NewFindingCollector(yield)
+		collector.Finally(clusters.WalkClusters(ctx, func(cluster *storage.Cluster) error {
+			expiry := cluster.GetStatus().GetCertExpiryStatus().GetSensorCertExpiry()
+			if expiry == nil {
+				// The cluster hasn't connected yet, so there is no cert to report on.
+				return nil
 			}
-			result, err := s.GetCertExpiry(ctx, &v1.GetCertExpiry_Request{
-				Component: v1.GetCertExpiry_Component(i),
+			return collector.Yield(&finding{
+				component:            securedClusterComponent,
+				name:                 cluster.GetName(),
+				hoursUntilExpiration: int(time.Until(expiry.AsTime()).Hours()),
 			})
-			if err != nil {
-				// Ignore particular component errors, as they do not affect
-				// other components metrics.
-				logging.LoggerForModule().Errorw("Failed to get certificate expiry",
-					logging.String("component", component), logging.Err(err))
-				continue
-			}
-			f.component = component
-			if result != nil {
-				f.hoursUntilExpiration = int(time.Until(result.GetExpiry().AsTime()).Hours())
-			}
-			if !yield(&f, nil) {
-				return
-			}
-		}
+		}))
 	}
 }

central/metrics/custom/expiry/tracker_test.go

@@ -4,12 +4,17 @@ import (
 	"context"
 	"errors"
 	"testing"
+	"time"
 
 	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
+	clusterDSMocks "github.com/stackrox/rox/central/cluster/datastore/mocks"
 	"github.com/stackrox/rox/central/credentialexpiry/service"
 	v1 "github.com/stackrox/rox/generated/api/v1"
+	"github.com/stackrox/rox/generated/storage"
 	"github.com/stretchr/testify/assert"
+	"go.uber.org/mock/gomock"
 	"google.golang.org/grpc"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 var errTest = errors.New("test")
@@ -39,11 +44,53 @@ var _ service.Service = (*mockService)(nil)
 
 func Test_track(t *testing.T) {
 	var s mockService
+	ctrl := gomock.NewController(t)
+	clusters := clusterDSMocks.NewMockDataStore(ctrl)
+	clusters.EXPECT().WalkClusters(gomock.Any(), gomock.Any()).Return(nil)
 
 	components := make([]string, 0, len(v1.GetCertExpiry_Component_name))
-	for f := range track(context.Background(), &s) {
+	for f := range track(context.Background(), &s, clusters) {
 		components = append(components, f.component)
 	}
 	assert.ElementsMatch(t, []string{"SCANNER", "CENTRAL_DB", "CENTRAL"}, components,
 		"should have no UNKNOWN and SCANNER_V4")
 }
+
+func Test_track_securedClusters(t *testing.T) {
+	var s mockService
+	ctrl := gomock.NewController(t)
+	clusters := clusterDSMocks.NewMockDataStore(ctrl)
+
+	connected := &storage.Cluster{
+		Name: "connected-cluster",
+		Status: &storage.ClusterStatus{
+			CertExpiryStatus: &storage.ClusterCertExpiryStatus{
+				SensorCertExpiry: timestamppb.New(time.Now().Add(24 * time.Hour)),
+			},
+		},
+	}
+	neverConnected := &storage.Cluster{Name: "never-connected-cluster"}
+
+	clusters.EXPECT().WalkClusters(gomock.Any(), gomock.Any()).
+		DoAndReturn(func(_ context.Context, fn func(*storage.Cluster) error) error {
+			for _, cluster := range []*storage.Cluster{connected, neverConnected} {
+				if err := fn(cluster); err != nil {
+					return err
+				}
+			}
+			return nil
+		})
+
+	var securedClusterFindings []*finding
+	for f, err := range track(context.Background(), &s, clusters) {
+		assert.NoError(t, err)
+		if f.component == securedClusterComponent {
+			securedClusterFindings = append(securedClusterFindings, f)
+		}
+	}
+
+	if assert.Len(t, securedClusterFindings, 1, "the never-connected cluster should not produce a finding") {
+		assert.Equal(t, "connected-cluster", securedClusterFindings[0].name)
+		assert.InDelta(t, 24, securedClusterFindings[0].hoursUntilExpiration, 1)
+	}
+}

central/metrics/custom/runner.go

@@ -95,7 +95,7 @@ func makeRunner(ds *runnerDatastores) trackerRunner {
 			"total_policies": policies.LazyLabels.GetLabels(),
 		}),
 	}, {
-		expiry.New(ds.expiry),
+		expiry.New(ds.expiry, ds.clusters),
 		withHardcodedConfiguration(60, map[string][]string{
 			// rox_central_cert_exp_hours
 			"hours": expiry.LazyLabels.GetLabels(),