diff --git a/admission/rulebinding/cache/cache.go b/admission/rulebinding/cache/cache.go index 9f2cafe3..81bb9e94 100644 --- a/admission/rulebinding/cache/cache.go +++ b/admission/rulebinding/cache/cache.go @@ -19,6 +19,11 @@ import ( ) const ( + // IncludeClusterObjects is a label on a RuntimeRuleAlertBinding. A binding + // labelled includeClusterObjects: "false" applies to namespaced objects + // only; without the label it also applies to cluster-scoped objects. It is + // read from the binding, never from the admitted object, so whoever + // creates a cluster-scoped object cannot turn off its evaluation. IncludeClusterObjects = "includeClusterObjects" ) @@ -34,15 +39,19 @@ type RBCache struct { notifiers []*chan rulebindingmanager.RuleBindingNotify // when true, skip rule binding filtering and return all rules ignoreRuleBindings bool + // when true and rule bindings are ignored, return no rules for + // cluster-scoped objects. Set from the operator config. + excludeClusterObjects bool } -func NewCache(k8sClient k8sclient.K8sClientInterface, ruleCreator rules.RuleCreator, ignoreRuleBindings bool) *RBCache { +func NewCache(k8sClient k8sclient.K8sClientInterface, ruleCreator rules.RuleCreator, ignoreRuleBindings, excludeClusterObjects bool) *RBCache { return &RBCache{ - k8sClient: k8sClient, - ruleCreator: ruleCreator, - rbNameToRB: maps.SafeMap[string, typesv1.RuntimeAlertRuleBinding]{}, - watchResources: resourcesToWatch(), - ignoreRuleBindings: ignoreRuleBindings, + k8sClient: k8sClient, + ruleCreator: ruleCreator, + rbNameToRB: maps.SafeMap[string, typesv1.RuntimeAlertRuleBinding]{}, + watchResources: resourcesToWatch(), + ignoreRuleBindings: ignoreRuleBindings, + excludeClusterObjects: excludeClusterObjects, } } @@ -62,14 +71,14 @@ func (c *RBCache) WatchResources() []watcher.WatchResource { // ------------------ rulebindingmanager.RuleBindingCache methods ----------------------- func (c *RBCache) ListRulesForObject(ctx context.Context, object *unstructured.Unstructured) []rules.RuleEvaluator { + clusterObject := object.GetNamespace() == "" + if c.ignoreRuleBindings { - // rules update is enabled, apply all rules regardless of bindings, but still respect - // cluster object opt-out label to avoid surprising cluster-level enforcement. - if object.GetNamespace() == "" { - includeClusterObjects, ok := object.GetLabels()[IncludeClusterObjects] - if ok && includeClusterObjects == "false" { - return []rules.RuleEvaluator{} - } + // rules update is enabled, apply all rules regardless of bindings. The + // operator config, not the admitted object, decides whether cluster + // objects are evaluated. + if clusterObject && c.excludeClusterObjects { + return []rules.RuleEvaluator{} } return c.ruleCreator.CreateAllRules() } @@ -79,18 +88,12 @@ func (c *RBCache) ListRulesForObject(ctx context.Context, object *unstructured.U for _, rb := range c.rbNameToRB.Values() { rbName := uniqueName(&rb) - // check if the object is cluster object - if object.GetNamespace() == "" { - includeClusterObjects, ok := object.GetLabels()[IncludeClusterObjects] - if !ok { - includeClusterObjects = "true" - } - - if includeClusterObjects == "false" { - continue + // the binding, not the admitted object, decides whether it applies to + // cluster objects + if clusterObject { + if bindingIncludesClusterObjects(&rb) { + rbNames = append(rbNames, rbName) } - - rbNames = append(rbNames, rbName) continue } diff --git a/admission/rulebinding/cache/cache_test.go b/admission/rulebinding/cache/cache_test.go index 944659bc..779d2889 100644 --- a/admission/rulebinding/cache/cache_test.go +++ b/admission/rulebinding/cache/cache_test.go @@ -36,7 +36,7 @@ func TestNewCache(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { k8sAPI := utils.NewK8sInterfaceFake(nil) - cache := NewCache(k8sAPI, &rules.RuleCreatorMock{}, false) + cache := NewCache(k8sAPI, &rules.RuleCreatorMock{}, false, false) assert.NotNil(t, cache) assert.Equal(t, k8sAPI, cache.k8sClient) @@ -52,7 +52,7 @@ func TestCacheConcurrentAccess(t *testing.T) { new func() *RBCache }{ {name: "constructor", new: func() *RBCache { - return NewCache(nil, &rules.RuleCreatorMock{}, false) + return NewCache(nil, &rules.RuleCreatorMock{}, false, false) }}, {name: "partial literal", new: func() *RBCache { return &RBCache{ruleCreator: &rules.RuleCreatorMock{}} @@ -312,46 +312,133 @@ func TestHandlersIgnoreNonRuleBindingKinds(t *testing.T) { } func TestListRulesForObjectIgnoreBindings(t *testing.T) { - c := &RBCache{ - k8sClient: k8sinterface.NewKubernetesApiMock(), - ruleCreator: &rules.RuleCreatorMock{}, - rbNameToRules: maps.SafeMap[string, []rules.RuleEvaluator]{}, - ignoreRuleBindings: true, + newCache := func(excludeClusterObjects bool) *RBCache { + return NewCache(k8sinterface.NewKubernetesApiMock(), &rules.RuleCreatorMock{}, true, excludeClusterObjects) } - - t.Run("namespaced object returns all rules", func(t *testing.T) { - obj := &unstructured.Unstructured{ - Object: map[string]interface{}{ - "metadata": map[string]interface{}{ - "name": "any", - "namespace": "ns", + namespacedObject := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "any", + "namespace": "ns", + }, + }, + } + labelledClusterObject := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "cluster-obj", + "labels": map[string]interface{}{ + IncludeClusterObjects: "false", }, }, - } + }, + } - ruleEvaluators := c.ListRulesForObject(context.Background(), obj) - assert.Len(t, ruleEvaluators, 2) - assert.Equal(t, "rule-1", ruleEvaluators[0].ID()) - assert.Equal(t, "rule-2", ruleEvaluators[1].ID()) + t.Run("namespaced object returns all rules", func(t *testing.T) { + assert.Equal(t, []string{"rule-1", "rule-2"}, ruleIDs(newCache(false).ListRulesForObject(context.Background(), namespacedObject))) }) - t.Run("cluster object respects includeClusterObjects=false", func(t *testing.T) { - obj := &unstructured.Unstructured{ - Object: map[string]interface{}{ - "metadata": map[string]interface{}{ - "name": "cluster-obj", - "labels": map[string]interface{}{ - "includeClusterObjects": "false", - }, - }, + t.Run("cluster object label does not opt out", func(t *testing.T) { + assert.Equal(t, []string{"rule-1", "rule-2"}, ruleIDs(newCache(false).ListRulesForObject(context.Background(), labelledClusterObject))) + }) + + t.Run("operator config excludes cluster objects", func(t *testing.T) { + c := newCache(true) + clusterObject := &unstructured.Unstructured{} + clusterObject.SetName("cluster-obj") + assert.Empty(t, c.ListRulesForObject(context.Background(), clusterObject)) + assert.Equal(t, []string{"rule-1", "rule-2"}, ruleIDs(c.ListRulesForObject(context.Background(), namespacedObject))) + }) +} + +// TestListRulesForClusterObject verifies that the RuleBinding, not the +// admitted object, decides whether a binding applies to cluster-scoped objects. +func TestListRulesForClusterObject(t *testing.T) { + binding := func(name string, labels map[string]string, ruleID string) typesv1.RuntimeAlertRuleBinding { + return typesv1.RuntimeAlertRuleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: name, Labels: labels}, + Spec: typesv1.RuntimeAlertRuleBindingSpec{ + Rules: []typesv1.RuntimeAlertRuleBindingRule{{RuleID: ruleID}}, }, } + } + clusterObject := func(labels map[string]interface{}) *unstructured.Unstructured { + metadata := map[string]interface{}{"name": "cluster-admin-binding"} + if labels != nil { + metadata["labels"] = labels + } + return &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "ClusterRoleBinding", + "metadata": metadata, + }} + } + optOut := map[string]string{IncludeClusterObjects: "false"} - ruleEvaluators := c.ListRulesForObject(context.Background(), obj) - assert.Len(t, ruleEvaluators, 0) + tests := []struct { + name string + bindings []typesv1.RuntimeAlertRuleBinding + object *unstructured.Unstructured + expected []string + }{ + { + name: "binding applies to cluster objects by default", + bindings: []typesv1.RuntimeAlertRuleBinding{binding("all", nil, "R2000")}, + object: clusterObject(nil), + expected: []string{"R2000"}, + }, + { + name: "object label does not opt the object out", + bindings: []typesv1.RuntimeAlertRuleBinding{binding("all", nil, "R2000")}, + object: clusterObject(map[string]interface{}{IncludeClusterObjects: "false"}), + expected: []string{"R2000"}, + }, + { + name: "binding label opts that binding out", + bindings: []typesv1.RuntimeAlertRuleBinding{ + binding("all", nil, "R2000"), + binding("namespaced-only", optOut, "R2001"), + }, + object: clusterObject(nil), + expected: []string{"R2000"}, + }, + { + name: "binding label true keeps cluster objects", + bindings: []typesv1.RuntimeAlertRuleBinding{binding("all", map[string]string{IncludeClusterObjects: "true"}, "R2000")}, + object: clusterObject(map[string]interface{}{IncludeClusterObjects: "false"}), + expected: []string{"R2000"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := NewCacheMock() + for i := range tt.bindings { + c.addRuleBinding(&tt.bindings[i]) + } + assert.ElementsMatch(t, tt.expected, ruleIDs(c.ListRulesForObject(context.Background(), tt.object))) + }) + } + + t.Run("opted-out binding still applies to namespaced objects", func(t *testing.T) { + c := NewCacheMock() + rb := binding("namespaced-only", optOut, "R2001") + c.addRuleBinding(&rb) + pod := &unstructured.Unstructured{} + pod.SetName("pod") + pod.SetNamespace("default") + assert.Equal(t, []string{"R2001"}, ruleIDs(c.ListRulesForObject(context.Background(), pod))) }) } +func ruleIDs(evaluators []rules.RuleEvaluator) []string { + ids := make([]string, 0, len(evaluators)) + for _, e := range evaluators { + ids = append(ids, e.ID()) + } + return ids +} + func TestCreateRule(t *testing.T) { c := NewCacheMock() tests := []struct { diff --git a/admission/rulebinding/cache/helpers.go b/admission/rulebinding/cache/helpers.go index c6d23cd4..5ddedbe6 100644 --- a/admission/rulebinding/cache/helpers.go +++ b/admission/rulebinding/cache/helpers.go @@ -31,6 +31,12 @@ func namespaceListHasName(list *corev1.NamespaceList, name string) bool { // other CRDs (notably Rules) before attempting type conversion. const ruleBindingKind = "RuntimeRuleAlertBinding" +// bindingIncludesClusterObjects reports whether rb applies to cluster-scoped +// objects: it does unless the binding is labelled includeClusterObjects: "false". +func bindingIncludesClusterObjects(rb *typesv1.RuntimeAlertRuleBinding) bool { + return rb.GetLabels()[IncludeClusterObjects] != "false" +} + func uniqueName(obj metav1.Object) string { return utils.CreateK8sPodID(obj.GetNamespace(), obj.GetName()) } diff --git a/admission/rulesupdate/updater.go b/admission/rulesupdate/updater.go index ba4d0223..aa04b584 100644 --- a/admission/rulesupdate/updater.go +++ b/admission/rulesupdate/updater.go @@ -31,6 +31,10 @@ type RulesUpdaterConfig struct { Interval time.Duration `mapstructure:"interval"` Namespace string `mapstructure:"namespace"` Enabled bool `mapstructure:"enabled"` + // ExcludeClusterObjects stops admission rules from being evaluated + // against cluster-scoped objects while Enabled makes the admission + // controller ignore rule bindings and run every rule. + ExcludeClusterObjects bool `mapstructure:"excludeClusterObjects"` } func NewRulesUpdator(ctx context.Context, k8sClient *k8sinterface.KubernetesApi, config RulesUpdaterConfig) *RulesUpdater { diff --git a/config/config.go b/config/config.go index a23a9db2..a0f39398 100644 --- a/config/config.go +++ b/config/config.go @@ -471,6 +471,7 @@ func LoadConfig(path string) (Config, error) { viper.SetDefault("rulesUpdateConfig.enabled", false) viper.SetDefault("rulesUpdateConfig.interval", 5*time.Minute) viper.SetDefault("rulesUpdateConfig.namespace", "default") + viper.SetDefault("rulesUpdateConfig.excludeClusterObjects", false) // Node agent autoscaler defaults viper.SetDefault("nodeAgentAutoscaler.enabled", false) diff --git a/docs/features/cel-admission-rules.md b/docs/features/cel-admission-rules.md index 5eb4037f..388e1e47 100644 --- a/docs/features/cel-admission-rules.md +++ b/docs/features/cel-admission-rules.md @@ -106,6 +106,35 @@ Rules that target cluster-scoped or non-pod resources need a backend change (an admission-request identity in incident correlation) before they can produce incidents. +## Cluster-Scoped Objects + +`RBCache.ListRulesForObject` does not match `podSelector` or +`namespaceSelector` for cluster-scoped objects (ClusterRoleBinding, +ClusterRole, Namespace, webhook configurations, ...). Every +`RuntimeRuleAlertBinding` applies to them unless the binding is labelled +`includeClusterObjects: "false"`: + +```yaml +apiVersion: kubescape.io/v1 +kind: RuntimeRuleAlertBinding +metadata: + name: namespaced-only + labels: + includeClusterObjects: "false" +spec: + rules: + - ruleID: "R2000" +``` + +When `rulesUpdateConfig.enabled` is set, the operator ignores bindings and runs +every rule on every object. Set `rulesUpdateConfig.excludeClusterObjects` to +`true` in the operator's `config.json` to skip cluster-scoped objects in that +mode. It defaults to `false`. + +The label is read from the binding and the setting from the operator config, +never from the admitted object. A label on a ClusterRoleBinding or Namespace +has no effect, so whoever creates the object cannot turn off its evaluation. + ## Event Type Constant `admission/cel/cel.go` defines: diff --git a/main.go b/main.go index c42aa2d9..ab5df792 100644 --- a/main.go +++ b/main.go @@ -257,7 +257,7 @@ func main() { dWatcher := dynamicwatcher.NewWatchHandler(k8sApi, ksStorageClient.SpdxV1beta1(), operatorConfig.SkipNamespace) // create ruleBinding cache with injected rule creator - ruleBindingCache := rulebindingcachev1.NewCache(k8sApi, celRuleCreator, operatorConfig.RulesUpdateEnabled()) + ruleBindingCache := rulebindingcachev1.NewCache(k8sApi, celRuleCreator, operatorConfig.RulesUpdateEnabled(), cfg.RulesUpdateConfig.ExcludeClusterObjects) dWatcher.AddAdaptor(ruleBindingCache) // Rules watcher — syncs k8s-admission rules from CRDs to the creator