Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 27 additions & 24 deletions admission/rulebinding/cache/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,11 @@ import (
)

const (
// IncludeClusterObjects is a label on a RuntimeRuleAlertBinding. A binding
// labelled includeClusterObjects: "false" applies to namespaced objects
// only; without the label it also applies to cluster-scoped objects. It is
// read from the binding, never from the admitted object, so whoever
// creates a cluster-scoped object cannot turn off its evaluation.
IncludeClusterObjects = "includeClusterObjects"
)

Expand All @@ -34,15 +39,19 @@ type RBCache struct {
notifiers []*chan rulebindingmanager.RuleBindingNotify
// when true, skip rule binding filtering and return all rules
ignoreRuleBindings bool
// when true and rule bindings are ignored, return no rules for
// cluster-scoped objects. Set from the operator config.
excludeClusterObjects bool
}

func NewCache(k8sClient k8sclient.K8sClientInterface, ruleCreator rules.RuleCreator, ignoreRuleBindings bool) *RBCache {
func NewCache(k8sClient k8sclient.K8sClientInterface, ruleCreator rules.RuleCreator, ignoreRuleBindings, excludeClusterObjects bool) *RBCache {
return &RBCache{
k8sClient: k8sClient,
ruleCreator: ruleCreator,
rbNameToRB: maps.SafeMap[string, typesv1.RuntimeAlertRuleBinding]{},
watchResources: resourcesToWatch(),
ignoreRuleBindings: ignoreRuleBindings,
k8sClient: k8sClient,
ruleCreator: ruleCreator,
rbNameToRB: maps.SafeMap[string, typesv1.RuntimeAlertRuleBinding]{},
watchResources: resourcesToWatch(),
ignoreRuleBindings: ignoreRuleBindings,
excludeClusterObjects: excludeClusterObjects,
}
}

Expand All @@ -62,14 +71,14 @@ func (c *RBCache) WatchResources() []watcher.WatchResource {
// ------------------ rulebindingmanager.RuleBindingCache methods -----------------------

func (c *RBCache) ListRulesForObject(ctx context.Context, object *unstructured.Unstructured) []rules.RuleEvaluator {
clusterObject := object.GetNamespace() == ""

if c.ignoreRuleBindings {
// rules update is enabled, apply all rules regardless of bindings, but still respect
// cluster object opt-out label to avoid surprising cluster-level enforcement.
if object.GetNamespace() == "" {
includeClusterObjects, ok := object.GetLabels()[IncludeClusterObjects]
if ok && includeClusterObjects == "false" {
return []rules.RuleEvaluator{}
}
// rules update is enabled, apply all rules regardless of bindings. The
// operator config, not the admitted object, decides whether cluster
// objects are evaluated.
if clusterObject && c.excludeClusterObjects {
return []rules.RuleEvaluator{}
}
return c.ruleCreator.CreateAllRules()
}
Expand All @@ -79,18 +88,12 @@ func (c *RBCache) ListRulesForObject(ctx context.Context, object *unstructured.U

for _, rb := range c.rbNameToRB.Values() {
rbName := uniqueName(&rb)
// check if the object is cluster object
if object.GetNamespace() == "" {
includeClusterObjects, ok := object.GetLabels()[IncludeClusterObjects]
if !ok {
includeClusterObjects = "true"
}

if includeClusterObjects == "false" {
continue
// the binding, not the admitted object, decides whether it applies to
// cluster objects
if clusterObject {
if bindingIncludesClusterObjects(&rb) {
rbNames = append(rbNames, rbName)
}

rbNames = append(rbNames, rbName)
continue
}

Expand Down
147 changes: 117 additions & 30 deletions admission/rulebinding/cache/cache_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func TestNewCache(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
k8sAPI := utils.NewK8sInterfaceFake(nil)
cache := NewCache(k8sAPI, &rules.RuleCreatorMock{}, false)
cache := NewCache(k8sAPI, &rules.RuleCreatorMock{}, false, false)

assert.NotNil(t, cache)
assert.Equal(t, k8sAPI, cache.k8sClient)
Expand All @@ -52,7 +52,7 @@ func TestCacheConcurrentAccess(t *testing.T) {
new func() *RBCache
}{
{name: "constructor", new: func() *RBCache {
return NewCache(nil, &rules.RuleCreatorMock{}, false)
return NewCache(nil, &rules.RuleCreatorMock{}, false, false)
}},
{name: "partial literal", new: func() *RBCache {
return &RBCache{ruleCreator: &rules.RuleCreatorMock{}}
Expand Down Expand Up @@ -312,46 +312,133 @@ func TestHandlersIgnoreNonRuleBindingKinds(t *testing.T) {
}

func TestListRulesForObjectIgnoreBindings(t *testing.T) {
c := &RBCache{
k8sClient: k8sinterface.NewKubernetesApiMock(),
ruleCreator: &rules.RuleCreatorMock{},
rbNameToRules: maps.SafeMap[string, []rules.RuleEvaluator]{},
ignoreRuleBindings: true,
newCache := func(excludeClusterObjects bool) *RBCache {
return NewCache(k8sinterface.NewKubernetesApiMock(), &rules.RuleCreatorMock{}, true, excludeClusterObjects)
}

t.Run("namespaced object returns all rules", func(t *testing.T) {
obj := &unstructured.Unstructured{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"name": "any",
"namespace": "ns",
namespacedObject := &unstructured.Unstructured{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"name": "any",
"namespace": "ns",
},
},
}
labelledClusterObject := &unstructured.Unstructured{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"name": "cluster-obj",
"labels": map[string]interface{}{
IncludeClusterObjects: "false",
},
},
}
},
}

ruleEvaluators := c.ListRulesForObject(context.Background(), obj)
assert.Len(t, ruleEvaluators, 2)
assert.Equal(t, "rule-1", ruleEvaluators[0].ID())
assert.Equal(t, "rule-2", ruleEvaluators[1].ID())
t.Run("namespaced object returns all rules", func(t *testing.T) {
assert.Equal(t, []string{"rule-1", "rule-2"}, ruleIDs(newCache(false).ListRulesForObject(context.Background(), namespacedObject)))
})

t.Run("cluster object respects includeClusterObjects=false", func(t *testing.T) {
obj := &unstructured.Unstructured{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"name": "cluster-obj",
"labels": map[string]interface{}{
"includeClusterObjects": "false",
},
},
t.Run("cluster object label does not opt out", func(t *testing.T) {
assert.Equal(t, []string{"rule-1", "rule-2"}, ruleIDs(newCache(false).ListRulesForObject(context.Background(), labelledClusterObject)))
})

t.Run("operator config excludes cluster objects", func(t *testing.T) {
c := newCache(true)
clusterObject := &unstructured.Unstructured{}
clusterObject.SetName("cluster-obj")
assert.Empty(t, c.ListRulesForObject(context.Background(), clusterObject))
assert.Equal(t, []string{"rule-1", "rule-2"}, ruleIDs(c.ListRulesForObject(context.Background(), namespacedObject)))
})
}

// TestListRulesForClusterObject verifies that the RuleBinding, not the
// admitted object, decides whether a binding applies to cluster-scoped objects.
func TestListRulesForClusterObject(t *testing.T) {
binding := func(name string, labels map[string]string, ruleID string) typesv1.RuntimeAlertRuleBinding {
return typesv1.RuntimeAlertRuleBinding{
ObjectMeta: metav1.ObjectMeta{Name: name, Labels: labels},
Spec: typesv1.RuntimeAlertRuleBindingSpec{
Rules: []typesv1.RuntimeAlertRuleBindingRule{{RuleID: ruleID}},
},
}
}
clusterObject := func(labels map[string]interface{}) *unstructured.Unstructured {
metadata := map[string]interface{}{"name": "cluster-admin-binding"}
if labels != nil {
metadata["labels"] = labels
}
return &unstructured.Unstructured{Object: map[string]interface{}{
"apiVersion": "rbac.authorization.k8s.io/v1",
"kind": "ClusterRoleBinding",
"metadata": metadata,
}}
}
optOut := map[string]string{IncludeClusterObjects: "false"}

ruleEvaluators := c.ListRulesForObject(context.Background(), obj)
assert.Len(t, ruleEvaluators, 0)
tests := []struct {
name string
bindings []typesv1.RuntimeAlertRuleBinding
object *unstructured.Unstructured
expected []string
}{
{
name: "binding applies to cluster objects by default",
bindings: []typesv1.RuntimeAlertRuleBinding{binding("all", nil, "R2000")},
object: clusterObject(nil),
expected: []string{"R2000"},
},
{
name: "object label does not opt the object out",
bindings: []typesv1.RuntimeAlertRuleBinding{binding("all", nil, "R2000")},
object: clusterObject(map[string]interface{}{IncludeClusterObjects: "false"}),
expected: []string{"R2000"},
},
{
name: "binding label opts that binding out",
bindings: []typesv1.RuntimeAlertRuleBinding{
binding("all", nil, "R2000"),
binding("namespaced-only", optOut, "R2001"),
},
object: clusterObject(nil),
expected: []string{"R2000"},
},
{
name: "binding label true keeps cluster objects",
bindings: []typesv1.RuntimeAlertRuleBinding{binding("all", map[string]string{IncludeClusterObjects: "true"}, "R2000")},
object: clusterObject(map[string]interface{}{IncludeClusterObjects: "false"}),
expected: []string{"R2000"},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := NewCacheMock()
for i := range tt.bindings {
c.addRuleBinding(&tt.bindings[i])
}
assert.ElementsMatch(t, tt.expected, ruleIDs(c.ListRulesForObject(context.Background(), tt.object)))
})
}

t.Run("opted-out binding still applies to namespaced objects", func(t *testing.T) {
c := NewCacheMock()
rb := binding("namespaced-only", optOut, "R2001")
c.addRuleBinding(&rb)
pod := &unstructured.Unstructured{}
pod.SetName("pod")
pod.SetNamespace("default")
assert.Equal(t, []string{"R2001"}, ruleIDs(c.ListRulesForObject(context.Background(), pod)))
})
}

func ruleIDs(evaluators []rules.RuleEvaluator) []string {
ids := make([]string, 0, len(evaluators))
for _, e := range evaluators {
ids = append(ids, e.ID())
}
return ids
}

func TestCreateRule(t *testing.T) {
c := NewCacheMock()
tests := []struct {
Expand Down
6 changes: 6 additions & 0 deletions admission/rulebinding/cache/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@ func namespaceListHasName(list *corev1.NamespaceList, name string) bool {
// other CRDs (notably Rules) before attempting type conversion.
const ruleBindingKind = "RuntimeRuleAlertBinding"

// bindingIncludesClusterObjects reports whether rb applies to cluster-scoped
// objects: it does unless the binding is labelled includeClusterObjects: "false".
func bindingIncludesClusterObjects(rb *typesv1.RuntimeAlertRuleBinding) bool {
return rb.GetLabels()[IncludeClusterObjects] != "false"
}

func uniqueName(obj metav1.Object) string {
return utils.CreateK8sPodID(obj.GetNamespace(), obj.GetName())
}
Expand Down
4 changes: 4 additions & 0 deletions admission/rulesupdate/updater.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ type RulesUpdaterConfig struct {
Interval time.Duration `mapstructure:"interval"`
Namespace string `mapstructure:"namespace"`
Enabled bool `mapstructure:"enabled"`
// ExcludeClusterObjects stops admission rules from being evaluated
// against cluster-scoped objects while Enabled makes the admission
// controller ignore rule bindings and run every rule.
ExcludeClusterObjects bool `mapstructure:"excludeClusterObjects"`
}

func NewRulesUpdator(ctx context.Context, k8sClient *k8sinterface.KubernetesApi, config RulesUpdaterConfig) *RulesUpdater {
Expand Down
1 change: 1 addition & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -471,6 +471,7 @@ func LoadConfig(path string) (Config, error) {
viper.SetDefault("rulesUpdateConfig.enabled", false)
viper.SetDefault("rulesUpdateConfig.interval", 5*time.Minute)
viper.SetDefault("rulesUpdateConfig.namespace", "default")
viper.SetDefault("rulesUpdateConfig.excludeClusterObjects", false)

// Node agent autoscaler defaults
viper.SetDefault("nodeAgentAutoscaler.enabled", false)
Expand Down
29 changes: 29 additions & 0 deletions docs/features/cel-admission-rules.md
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,35 @@ Rules that target cluster-scoped or non-pod resources need a backend change
(an admission-request identity in incident correlation) before they can
produce incidents.

## Cluster-Scoped Objects

`RBCache.ListRulesForObject` does not match `podSelector` or
`namespaceSelector` for cluster-scoped objects (ClusterRoleBinding,
ClusterRole, Namespace, webhook configurations, ...). Every
`RuntimeRuleAlertBinding` applies to them unless the binding is labelled
`includeClusterObjects: "false"`:

```yaml
apiVersion: kubescape.io/v1
kind: RuntimeRuleAlertBinding
metadata:
name: namespaced-only
labels:
includeClusterObjects: "false"
spec:
rules:
- ruleID: "R2000"
```

When `rulesUpdateConfig.enabled` is set, the operator ignores bindings and runs
every rule on every object. Set `rulesUpdateConfig.excludeClusterObjects` to
`true` in the operator's `config.json` to skip cluster-scoped objects in that
mode. It defaults to `false`.

The label is read from the binding and the setting from the operator config,
never from the admitted object. A label on a ClusterRoleBinding or Namespace
has no effect, so whoever creates the object cannot turn off its evaluation.

## Event Type Constant

`admission/cel/cel.go` defines:
Expand Down
2 changes: 1 addition & 1 deletion main.go
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ func main() {
dWatcher := dynamicwatcher.NewWatchHandler(k8sApi, ksStorageClient.SpdxV1beta1(), operatorConfig.SkipNamespace)

// create ruleBinding cache with injected rule creator
ruleBindingCache := rulebindingcachev1.NewCache(k8sApi, celRuleCreator, operatorConfig.RulesUpdateEnabled())
ruleBindingCache := rulebindingcachev1.NewCache(k8sApi, celRuleCreator, operatorConfig.RulesUpdateEnabled(), cfg.RulesUpdateConfig.ExcludeClusterObjects)
dWatcher.AddAdaptor(ruleBindingCache)

// Rules watcher — syncs k8s-admission rules from CRDs to the creator
Expand Down
Loading