Skip to content

Commit 9497b5d

Browse files
sradcocursoragent
andcommitted
test: add RBAC e2e tests for create and delete endpoints
Add e2e tests verifying that the management API enforces Kubernetes RBAC for create and delete alert rule operations. Three user profiles are tested: unprivileged (expects 403), namespace-scoped (succeeds in own namespace, denied elsewhere), and cluster-admin (succeeds everywhere). Also fixes a critical bug in newUserScopedClientsets: when the base rest.Config uses client certificates (common in CI kubeconfigs), CopyConfig preserved them. Since Kubernetes authenticates via client certs when both certs and bearer token are present, user RBAC was bypassed entirely. Clear CertData/CertFile/KeyData/KeyFile so the API server authenticates exclusively via the user's bearer token. Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent 7dd83cf commit 9497b5d

6 files changed

Lines changed: 597 additions & 4 deletions

File tree

internal/managementrouter/router.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111

1212
"github.com/gorilla/mux"
1313
"github.com/sirupsen/logrus"
14+
apierrors "k8s.io/apimachinery/pkg/api/errors"
1415

1516
"github.com/openshift/monitoring-plugin/pkg/k8s"
1617
"github.com/openshift/monitoring-plugin/pkg/management"
@@ -62,6 +63,7 @@ func authMiddleware(next http.Handler) http.Handler {
6263
})
6364
}
6465

66+
// writeError sends a JSON {"error": message} response with the given status code.
6567
func writeError(w http.ResponseWriter, statusCode int, message string) {
6668
w.Header().Set("Content-Type", "application/json")
6769
w.WriteHeader(statusCode)
@@ -75,11 +77,15 @@ func writeError(w http.ResponseWriter, statusCode int, message string) {
7577
}
7678
}
7779

80+
// handleError maps err to an HTTP status via parseError and writes the response.
7881
func handleError(w http.ResponseWriter, err error) {
7982
status, message := parseError(err)
8083
writeError(w, status, message)
8184
}
8285

86+
// parseError inspects err and returns a (statusCode, userMessage) pair.
87+
// Domain errors are mapped to 4xx; Kubernetes Unauthorized/Forbidden are mapped
88+
// to 401/403 with generic messages to avoid leaking API server details.
8389
func parseError(err error) (int, string) {
8490
var nf *management.NotFoundError
8591
if errors.As(err, &nf) {
@@ -97,6 +103,12 @@ func parseError(err error) (int, string) {
97103
if errors.As(err, &ce) {
98104
return http.StatusConflict, err.Error()
99105
}
106+
if apierrors.IsUnauthorized(err) {
107+
return http.StatusUnauthorized, "authentication failed"
108+
}
109+
if apierrors.IsForbidden(err) {
110+
return http.StatusForbidden, "insufficient permissions"
111+
}
100112
log.WithError(err).Error("unexpected management API error")
101113
return http.StatusInternalServerError, "An unexpected error occurred"
102114
}
Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
package managementrouter
2+
3+
import (
4+
"fmt"
5+
"net/http"
6+
"testing"
7+
8+
apierrors "k8s.io/apimachinery/pkg/api/errors"
9+
"k8s.io/apimachinery/pkg/runtime/schema"
10+
11+
"github.com/openshift/monitoring-plugin/pkg/management"
12+
)
13+
14+
func TestParseError(t *testing.T) {
15+
tests := []struct {
16+
name string
17+
err error
18+
expectedStatus int
19+
expectedMsg string
20+
}{
21+
{
22+
name: "NotFoundError",
23+
err: &management.NotFoundError{Resource: "AlertRule", Id: "abc"},
24+
expectedStatus: http.StatusNotFound,
25+
},
26+
{
27+
name: "ValidationError",
28+
err: &management.ValidationError{Message: "bad input"},
29+
expectedStatus: http.StatusBadRequest,
30+
},
31+
{
32+
name: "NotAllowedError",
33+
err: &management.NotAllowedError{Message: "not allowed"},
34+
expectedStatus: http.StatusMethodNotAllowed,
35+
},
36+
{
37+
name: "ConflictError",
38+
err: &management.ConflictError{Message: "conflict"},
39+
expectedStatus: http.StatusConflict,
40+
},
41+
{
42+
name: "Kubernetes Forbidden",
43+
err: apierrors.NewForbidden(schema.GroupResource{
44+
Group: "monitoring.coreos.com", Resource: "prometheusrules",
45+
}, "test-pr", fmt.Errorf("access denied")),
46+
expectedStatus: http.StatusForbidden,
47+
expectedMsg: "insufficient permissions",
48+
},
49+
{
50+
name: "Kubernetes Forbidden wrapped",
51+
err: fmt.Errorf("failed to get PrometheusRule: %w",
52+
apierrors.NewForbidden(schema.GroupResource{
53+
Group: "monitoring.coreos.com", Resource: "prometheusrules",
54+
}, "test-pr", fmt.Errorf("access denied"))),
55+
expectedStatus: http.StatusForbidden,
56+
expectedMsg: "insufficient permissions",
57+
},
58+
{
59+
name: "Kubernetes Unauthorized",
60+
err: apierrors.NewUnauthorized("token expired"),
61+
expectedStatus: http.StatusUnauthorized,
62+
expectedMsg: "authentication failed",
63+
},
64+
{
65+
name: "unknown error",
66+
err: fmt.Errorf("something unexpected"),
67+
expectedStatus: http.StatusInternalServerError,
68+
expectedMsg: "An unexpected error occurred",
69+
},
70+
}
71+
72+
for _, tt := range tests {
73+
t.Run(tt.name, func(t *testing.T) {
74+
status, msg := parseError(tt.err)
75+
if status != tt.expectedStatus {
76+
t.Errorf("expected status %d, got %d", tt.expectedStatus, status)
77+
}
78+
if tt.expectedMsg != "" && msg != tt.expectedMsg {
79+
t.Errorf("expected message %q, got %q", tt.expectedMsg, msg)
80+
}
81+
})
82+
}
83+
}

pkg/k8s/user_scoped_client.go

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,21 @@ type userScopedClientsets struct {
1313
osmV1 *osmv1client.Clientset
1414
}
1515

16+
// buildUserScopedConfig creates a rest.Config that authenticates exclusively
17+
// with the given bearer token. It uses AnonymousClientConfig to strip all
18+
// existing auth (certs, basic auth, auth/exec providers, impersonation) while
19+
// preserving the server connection settings (host, TLS CA, proxy).
20+
func buildUserScopedConfig(baseConfig *rest.Config, userToken string) *rest.Config {
21+
cfg := rest.AnonymousClientConfig(baseConfig)
22+
cfg.BearerToken = userToken
23+
return cfg
24+
}
25+
1626
// newUserScopedClientsets creates clientsets that carry the supplied bearer
1727
// token so that Kubernetes RBAC is enforced for the requesting user on all
1828
// mutating API calls.
1929
func newUserScopedClientsets(baseConfig *rest.Config, userToken string) (*userScopedClientsets, error) {
20-
cfg := rest.CopyConfig(baseConfig)
21-
// Override any SA token loaded from the file system with the user's token.
22-
cfg.BearerToken = userToken
23-
cfg.BearerTokenFile = ""
30+
cfg := buildUserScopedConfig(baseConfig, userToken)
2431

2532
monClient, err := monitoringv1client.NewForConfig(cfg)
2633
if err != nil {

pkg/k8s/user_scoped_client_test.go

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
package k8s
2+
3+
import (
4+
"testing"
5+
6+
"k8s.io/client-go/rest"
7+
)
8+
9+
func TestBuildUserScopedConfig(t *testing.T) {
10+
base := &rest.Config{
11+
Host: "https://api.example.com:6443",
12+
BearerToken: "sa-token",
13+
BearerTokenFile: "/var/run/secrets/kubernetes.io/serviceaccount/token",
14+
TLSClientConfig: rest.TLSClientConfig{
15+
Insecure: true,
16+
CertData: []byte("admin-cert"),
17+
KeyData: []byte("admin-key"),
18+
CertFile: "/path/to/cert",
19+
KeyFile: "/path/to/key",
20+
},
21+
}
22+
23+
cfg := buildUserScopedConfig(base, "user-token")
24+
25+
// Derived config uses the user token exclusively.
26+
if cfg.BearerToken != "user-token" {
27+
t.Errorf("derived BearerToken = %q, want %q", cfg.BearerToken, "user-token")
28+
}
29+
if cfg.BearerTokenFile != "" {
30+
t.Errorf("derived BearerTokenFile = %q, want empty", cfg.BearerTokenFile)
31+
}
32+
if cfg.CertData != nil {
33+
t.Error("derived CertData should be nil")
34+
}
35+
if cfg.KeyData != nil {
36+
t.Error("derived KeyData should be nil")
37+
}
38+
if cfg.CertFile != "" {
39+
t.Errorf("derived CertFile = %q, want empty", cfg.CertFile)
40+
}
41+
if cfg.KeyFile != "" {
42+
t.Errorf("derived KeyFile = %q, want empty", cfg.KeyFile)
43+
}
44+
if !cfg.Insecure {
45+
t.Error("derived Insecure should be preserved as true")
46+
}
47+
if cfg.Host != base.Host {
48+
t.Errorf("derived Host = %q, want %q", cfg.Host, base.Host)
49+
}
50+
51+
// Base config must not be mutated.
52+
if base.CertData == nil {
53+
t.Error("base CertData was mutated")
54+
}
55+
if base.KeyData == nil {
56+
t.Error("base KeyData was mutated")
57+
}
58+
if base.BearerToken != "sa-token" {
59+
t.Errorf("base BearerToken = %q, want %q", base.BearerToken, "sa-token")
60+
}
61+
if base.BearerTokenFile != "/var/run/secrets/kubernetes.io/serviceaccount/token" {
62+
t.Errorf("base BearerTokenFile = %q, was mutated", base.BearerTokenFile)
63+
}
64+
}

test/e2e/framework/framework.go

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,8 @@ func (f *Framework) HTTPClient() *http.Client {
113113
return f.httpClient
114114
}
115115

116+
// CreateNamespace creates a uniquely-named namespace (prefixed with name and
117+
// a unix timestamp) and returns the actual name plus a cleanup function.
116118
func (f *Framework) CreateNamespace(ctx context.Context, name string, isClusterMonitoringNamespace bool) (string, CleanupFunc, error) {
117119
testNamespace := fmt.Sprintf("%s-%d", name, time.Now().Unix())
118120
namespace := &corev1.Namespace{
@@ -212,3 +214,102 @@ func createServiceAccountToken(clientset *kubernetes.Clientset) (string, error)
212214
}
213215
return resp.Status.Token, nil
214216
}
217+
// ScopedUser represents a ServiceAccount with specific RBAC permissions for testing.
218+
type ScopedUser struct {
219+
Token string
220+
Cleanup CleanupFunc
221+
}
222+
223+
// requestServiceAccountToken creates a short-lived (1 hour) bearer token for
224+
// the named ServiceAccount via the TokenRequest API.
225+
func (f *Framework) requestServiceAccountToken(ctx context.Context, namespace, name string) (string, error) {
226+
expSeconds := int64(3600)
227+
treq := &authv1.TokenRequest{
228+
Spec: authv1.TokenRequestSpec{ExpirationSeconds: &expSeconds},
229+
}
230+
tokenResp, err := f.Clientset.CoreV1().ServiceAccounts(namespace).CreateToken(ctx, name, treq, metav1.CreateOptions{})
231+
if err != nil {
232+
return "", fmt.Errorf("requesting token for %s/%s: %w", namespace, name, err)
233+
}
234+
return tokenResp.Status.Token, nil
235+
}
236+
237+
// CreateScopedUser creates a ServiceAccount in the given namespace with a Role
238+
// granting the specified verbs on the specified resources. Returns a bearer token
239+
// and a cleanup function. The apiGroup should be e.g. "monitoring.coreos.com".
240+
func (f *Framework) CreateScopedUser(ctx context.Context, name, namespace, apiGroup string, resources, verbs []string) (*ScopedUser, error) {
241+
rollback := func() {
242+
_ = f.Clientset.RbacV1().RoleBindings(namespace).Delete(ctx, name, metav1.DeleteOptions{})
243+
_ = f.Clientset.RbacV1().Roles(namespace).Delete(ctx, name, metav1.DeleteOptions{})
244+
_ = f.Clientset.CoreV1().ServiceAccounts(namespace).Delete(ctx, name, metav1.DeleteOptions{})
245+
}
246+
247+
sa := &corev1.ServiceAccount{
248+
ObjectMeta: metav1.ObjectMeta{Name: name},
249+
}
250+
if _, err := f.Clientset.CoreV1().ServiceAccounts(namespace).Create(ctx, sa, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) {
251+
return nil, fmt.Errorf("creating service account %s/%s: %w", namespace, name, err)
252+
}
253+
254+
role := &rbacv1.Role{
255+
ObjectMeta: metav1.ObjectMeta{Name: name},
256+
Rules: []rbacv1.PolicyRule{{
257+
APIGroups: []string{apiGroup},
258+
Resources: resources,
259+
Verbs: verbs,
260+
}},
261+
}
262+
if _, err := f.Clientset.RbacV1().Roles(namespace).Create(ctx, role, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) {
263+
rollback()
264+
return nil, fmt.Errorf("creating role %s/%s: %w", namespace, name, err)
265+
}
266+
267+
rb := &rbacv1.RoleBinding{
268+
ObjectMeta: metav1.ObjectMeta{Name: name},
269+
Subjects: []rbacv1.Subject{{
270+
Kind: rbacv1.ServiceAccountKind,
271+
Name: name,
272+
Namespace: namespace,
273+
}},
274+
RoleRef: rbacv1.RoleRef{
275+
APIGroup: rbacv1.GroupName,
276+
Kind: "Role",
277+
Name: name,
278+
},
279+
}
280+
if _, err := f.Clientset.RbacV1().RoleBindings(namespace).Create(ctx, rb, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) {
281+
rollback()
282+
return nil, fmt.Errorf("creating role binding %s/%s: %w", namespace, name, err)
283+
}
284+
285+
token, err := f.requestServiceAccountToken(ctx, namespace, name)
286+
if err != nil {
287+
rollback()
288+
return nil, err
289+
}
290+
291+
return &ScopedUser{Token: token, Cleanup: func() error { rollback(); return nil }}, nil
292+
}
293+
294+
// CreateUnprivilegedUser creates a ServiceAccount with no RBAC permissions.
295+
func (f *Framework) CreateUnprivilegedUser(ctx context.Context, name, namespace string) (*ScopedUser, error) {
296+
sa := &corev1.ServiceAccount{
297+
ObjectMeta: metav1.ObjectMeta{Name: name},
298+
}
299+
if _, err := f.Clientset.CoreV1().ServiceAccounts(namespace).Create(ctx, sa, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) {
300+
return nil, fmt.Errorf("creating service account %s/%s: %w", namespace, name, err)
301+
}
302+
303+
token, err := f.requestServiceAccountToken(ctx, namespace, name)
304+
if err != nil {
305+
_ = f.Clientset.CoreV1().ServiceAccounts(namespace).Delete(ctx, name, metav1.DeleteOptions{})
306+
return nil, err
307+
}
308+
309+
cleanup := func() error {
310+
_ = f.Clientset.CoreV1().ServiceAccounts(namespace).Delete(ctx, name, metav1.DeleteOptions{})
311+
return nil
312+
}
313+
314+
return &ScopedUser{Token: token, Cleanup: cleanup}, nil
315+
}

0 commit comments

Comments
 (0)