From ef73b6f1125a3f357a90917c79187e567d4ebacb Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Wed, 26 Aug 2026 15:50:44 +0200 Subject: [PATCH] re-enable parallel kcp tests This makes it so that the clusters (workspaces) used by each test are not hardcoded, but based on the test's name (i.e. each test gets its own root cluster at "root::kbind"). This makes the tests independent of each other. On-behalf-of: @SAP christoph.mewes@sap.com Signed-off-by: Christoph Mewes --- contrib/kcp/bootstrap/config.go | 2 +- .../kcp/bootstrap/config/config/bootstrap.go | 11 +-- .../config/clusterworkspace-kube-bind.yaml | 2 +- .../kcp/bootstrap/config/core/bootstrap.go | 14 ++-- contrib/kcp/bootstrap/options/options.go | 14 +++- contrib/kcp/bootstrap/server.go | 6 ++ contrib/kcp/deploy/bootstrap.go | 18 ++--- contrib/kcp/test/e2e/backend.go | 6 +- contrib/kcp/test/e2e/kcp.go | 73 +++++++++++++++++-- contrib/kcp/test/e2e/kcp_3tier_test.go | 34 ++++++--- contrib/kcp/test/e2e/kcp_test.go | 41 +++++++---- test/e2e/bind/happy-case_test.go | 2 +- test/e2e/framework/dex.go | 2 +- test/e2e/framework/env.go | 33 ++++++++- test/e2e/framework/kcp.go | 4 +- test/e2e/framework/kubeconfig.go | 13 +++- 16 files changed, 199 insertions(+), 76 deletions(-) diff --git a/contrib/kcp/bootstrap/config.go b/contrib/kcp/bootstrap/config.go index cdb874344..710066984 100644 --- a/contrib/kcp/bootstrap/config.go +++ b/contrib/kcp/bootstrap/config.go @@ -53,7 +53,7 @@ func NewConfig(options *options.CompletedOptions) (*Config, error) { return nil, err } config.ClientConfig = rest.CopyConfig(config.ClientConfig) - config.ClientConfig = rest.AddUserAgent(config.ClientConfig, "kube-bind-kcp-init") + config.ClientConfig = rest.AddUserAgent(config.ClientConfig, "kbind-kcp-init") config.ClientConfig, err = newKCPRestConfig(config.ClientConfig) if err != nil { diff --git a/contrib/kcp/bootstrap/config/config/bootstrap.go b/contrib/kcp/bootstrap/config/config/bootstrap.go index aa1639a4b..f22c614fd 100644 --- a/contrib/kcp/bootstrap/config/config/bootstrap.go +++ b/contrib/kcp/bootstrap/config/config/bootstrap.go @@ -24,7 +24,6 @@ import ( kcpdynamic "github.com/kcp-dev/client-go/dynamic" confighelpers "github.com/kcp-dev/kcp/config/helpers" "github.com/kcp-dev/logicalcluster/v3" - "github.com/kcp-dev/sdk/apis/core" kcpclientset "github.com/kcp-dev/sdk/client/clientset/versioned/cluster" "k8s.io/apimachinery/pkg/util/sets" ) @@ -32,9 +31,6 @@ import ( //go:embed *.yaml var fs embed.FS -// RootClusterName is the workspace to host common faros APIs. -var RootClusterName = logicalcluster.NewPath("root") - // Bootstrap creates resources in this package by continuously retrying the list. // This is blocking, i.e. it only returns (with error) when the context is closed or with nil when // the bootstrapping is successfully completed. @@ -44,8 +40,9 @@ func Bootstrap( apiExtensionClusterClient kcpapiextensionsclientset.ClusterInterface, dynamicClusterClient kcpdynamic.ClusterInterface, batteriesIncluded sets.Set[string], + baseWorkspace logicalcluster.Path, ) error { - rootDiscoveryClient := apiExtensionClusterClient.Cluster(core.RootCluster.Path()).Discovery() - rootDynamicClient := dynamicClusterClient.Cluster(core.RootCluster.Path()) - return confighelpers.Bootstrap(ctx, rootDiscoveryClient, rootDynamicClient, batteriesIncluded, fs) + discoveryClient := apiExtensionClusterClient.Cluster(baseWorkspace).Discovery() + dynamicClient := dynamicClusterClient.Cluster(baseWorkspace) + return confighelpers.Bootstrap(ctx, discoveryClient, dynamicClient, batteriesIncluded, fs) } diff --git a/contrib/kcp/bootstrap/config/config/clusterworkspace-kube-bind.yaml b/contrib/kcp/bootstrap/config/config/clusterworkspace-kube-bind.yaml index 0212e1112..0cbdb7733 100644 --- a/contrib/kcp/bootstrap/config/config/clusterworkspace-kube-bind.yaml +++ b/contrib/kcp/bootstrap/config/config/clusterworkspace-kube-bind.yaml @@ -6,5 +6,5 @@ metadata: bootstrap.kcp.io/create-only: "true" spec: type: - name: organization + name: universal path: root diff --git a/contrib/kcp/bootstrap/config/core/bootstrap.go b/contrib/kcp/bootstrap/config/core/bootstrap.go index ec42d3806..6de3f5805 100644 --- a/contrib/kcp/bootstrap/config/core/bootstrap.go +++ b/contrib/kcp/bootstrap/config/core/bootstrap.go @@ -28,11 +28,6 @@ import ( "github.com/kube-bind/kube-bind/contrib/kcp/bootstrap/config/core/resources" ) -var ( - // KubeBindRootClusterName is the workspace to host common APIs. - KubeBindRootClusterName = logicalcluster.NewPath("root:kube-bind") -) - // Bootstrap creates resources in this package by continuously retrying the list. // This is blocking, i.e. it only returns (with error) when the context is closed or with nil when // the bootstrapping is successfully completed. @@ -42,10 +37,11 @@ func Bootstrap( apiExtensionClusterClient kcpapiextensionsclientset.ClusterInterface, dynamicClusterClient kcpdynamic.ClusterInterface, batteriesIncluded sets.Set[string], + baseWorkspace logicalcluster.Path, ) error { - computeDiscoveryClient := apiExtensionClusterClient.Cluster(KubeBindRootClusterName).Discovery() - computeDynamicClient := dynamicClusterClient.Cluster(KubeBindRootClusterName) + discoveryClient := apiExtensionClusterClient.Cluster(baseWorkspace).Discovery() + dynamicClient := dynamicClusterClient.Cluster(baseWorkspace) - crdClient := apiExtensionClusterClient.ApiextensionsV1().Cluster(KubeBindRootClusterName).CustomResourceDefinitions() - return resources.Bootstrap(ctx, kcpClientSet, computeDiscoveryClient, computeDynamicClient, crdClient, batteriesIncluded) + crdClient := apiExtensionClusterClient.ApiextensionsV1().Cluster(baseWorkspace).CustomResourceDefinitions() + return resources.Bootstrap(ctx, kcpClientSet, discoveryClient, dynamicClient, crdClient, batteriesIncluded) } diff --git a/contrib/kcp/bootstrap/options/options.go b/contrib/kcp/bootstrap/options/options.go index 95870c331..1e9c75607 100644 --- a/contrib/kcp/bootstrap/options/options.go +++ b/contrib/kcp/bootstrap/options/options.go @@ -19,6 +19,7 @@ package options import ( "fmt" + "github.com/kcp-dev/sdk/apis/core" "github.com/spf13/pflag" "k8s.io/component-base/logs" logsv1 "k8s.io/component-base/logs/api/v1" @@ -29,9 +30,13 @@ type Options struct { ExtraOptions } + type ExtraOptions struct { KCPKubeConfig string KCPContext string + + RootWorkspace string + KBindWorkspace string } type completedOptions struct { @@ -50,8 +55,11 @@ func NewOptions() *Options { logs.Verbosity = logsv1.VerbosityLevel(2) return &Options{ - Logs: logs, - ExtraOptions: ExtraOptions{}, + Logs: logs, + ExtraOptions: ExtraOptions{ + RootWorkspace: core.RootCluster.Path().String(), + KBindWorkspace: "root:kbind", + }, } } @@ -60,6 +68,8 @@ func (options *Options) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&options.KCPKubeConfig, "kcp-kubeconfig", options.KCPKubeConfig, "path to a kcp kubeconfig. Required to bootstrap the server.") fs.StringVar(&options.KCPContext, "context", options.KCPContext, "Name of the context in the kcp kubeconfig file to use") + fs.StringVar(&options.RootWorkspace, "root-workspace", options.RootWorkspace, "Workspace path to use as the root (must exist already)") + fs.StringVar(&options.KBindWorkspace, "kbind-workspace", options.KBindWorkspace, "kbind workspace to use (must exist already)") } func (options *Options) Complete() (*CompletedOptions, error) { diff --git a/contrib/kcp/bootstrap/server.go b/contrib/kcp/bootstrap/server.go index 773d5a5a1..c50c68fe3 100644 --- a/contrib/kcp/bootstrap/server.go +++ b/contrib/kcp/bootstrap/server.go @@ -19,6 +19,7 @@ package bootstrap import ( "context" + "github.com/kcp-dev/logicalcluster/v3" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" @@ -49,17 +50,21 @@ func (s *Server) Start(ctx context.Context) error { s.Config.ApiextensionsClient, s.Config.DynamicClusterClient, fakeBatteries, + logicalcluster.NewPath(s.Config.Options.ExtraOptions.RootWorkspace), ); err != nil { logger.Error(err, "failed to bootstrap initial config workspace") return nil // don't klog.Fatal. This only happens when context is cancelled. } + kbindWorkspace := logicalcluster.NewPath(s.Config.Options.ExtraOptions.KBindWorkspace) + if err := bootstrapcore.Bootstrap( ctx, s.Config.KcpClusterClient, s.Config.ApiextensionsClient, s.Config.DynamicClusterClient, fakeBatteries, + kbindWorkspace, ); err != nil { logger.Error(err, "failed to bootstrap core workspace") return nil // don't klog.Fatal. This only happens when context is cancelled. @@ -71,6 +76,7 @@ func (s *Server) Start(ctx context.Context) error { s.Config.ApiextensionsClient, s.Config.DynamicClusterClient, fakeBatteries, + kbindWorkspace, ); err != nil { logger.Error(err, "failed to bootstrap workspace") return nil // don't klog.Fatal. This only happens when context is cancelled. diff --git a/contrib/kcp/deploy/bootstrap.go b/contrib/kcp/deploy/bootstrap.go index 5808e12ad..af499bd65 100644 --- a/contrib/kcp/deploy/bootstrap.go +++ b/contrib/kcp/deploy/bootstrap.go @@ -39,11 +39,6 @@ import ( //go:embed examples/*.yaml var Examples embed.FS -var ( - // KubeBindRootClusterName is the workspace to host common APIs. - KubeBindRootClusterName = logicalcluster.NewPath("root:kube-bind") -) - // Bootstrap creates resources in this package by continuously retrying the list. // This is blocking, i.e. it only returns (with error) when the context is closed or with nil when // the bootstrapping is successfully completed. @@ -53,21 +48,22 @@ func Bootstrap( apiExtensionClusterClient kcpapiextensionsclientset.ClusterInterface, dynamicClusterClient kcpdynamic.ClusterInterface, batteriesIncluded sets.Set[string], + baseWorkspace logicalcluster.Path, ) error { - computeDiscoveryClient := apiExtensionClusterClient.Cluster(KubeBindRootClusterName).Discovery() - computeDynamicClient := dynamicClusterClient.Cluster(KubeBindRootClusterName) + discoveryClient := apiExtensionClusterClient.Cluster(baseWorkspace).Discovery() + dynamicClient := dynamicClusterClient.Cluster(baseWorkspace) - crdClient := apiExtensionClusterClient.ApiextensionsV1().Cluster(KubeBindRootClusterName).CustomResourceDefinitions() - kcpClient := kcpClientSet.Cluster(KubeBindRootClusterName) + crdClient := apiExtensionClusterClient.ApiextensionsV1().Cluster(baseWorkspace).CustomResourceDefinitions() + kcpClient := kcpClientSet.Cluster(baseWorkspace) - err := resources.Bootstrap(ctx, kcpClientSet, computeDiscoveryClient, computeDynamicClient, crdClient, batteriesIncluded) + err := resources.Bootstrap(ctx, kcpClientSet, discoveryClient, dynamicClient, crdClient, batteriesIncluded) if err != nil { return err } // create recursive apibinding so we can start controllers. // this is a temporary solution until we have a better way to bootstrap controllers. - return bindAPIExport(ctx, kcpClient, "kube-bind.io", KubeBindRootClusterName) + return bindAPIExport(ctx, kcpClient, "kube-bind.io", baseWorkspace) } func bindAPIExport(ctx context.Context, kcpClient kcpclient.Interface, exportName string, clusterPath logicalcluster.Path) error { diff --git a/contrib/kcp/test/e2e/backend.go b/contrib/kcp/test/e2e/backend.go index ff033191c..8306651ef 100644 --- a/contrib/kcp/test/e2e/backend.go +++ b/contrib/kcp/test/e2e/backend.go @@ -35,7 +35,7 @@ import ( "github.com/kube-bind/kube-bind/test/e2e/framework" ) -func bootstrapBackend(t *testing.T, rest *rest.Config, scope kubebindv1alpha2.InformerScope) string { +func bootstrapBackend(t *testing.T, rest *rest.Config, scope kubebindv1alpha2.InformerScope, kbindPath logicalcluster.Path) string { t.Helper() t.Log("Bootstrapping backend") @@ -46,7 +46,7 @@ func bootstrapBackend(t *testing.T, rest *rest.Config, scope kubebindv1alpha2.In exportUrl := "" kcptestinghelpers.Eventually(t, func() (bool, string) { - exportES, err := client.Cluster(logicalcluster.NewPath("root").Join("kube-bind")). + exportES, err := client.Cluster(kbindPath). ApisV1alpha1(). APIExportEndpointSlices(). Get(t.Context(), "kube-bind.io", metav1.GetOptions{}) @@ -61,7 +61,7 @@ func bootstrapBackend(t *testing.T, rest *rest.Config, scope kubebindv1alpha2.In }, wait.ForeverTestTimeout, time.Millisecond*100) require.NotEmpty(t, exportUrl, "APIExportEndpointSlice URL is empty") - _, backendKubeconfig := wsConfig(t, rest, logicalcluster.NewPath("root").Join("kube-bind")) + _, backendKubeconfig := wsConfig(t, rest, kbindPath) t.Log("Starting kube-bind backend for kcp") addr, _ := framework.StartBackend(t, diff --git a/contrib/kcp/test/e2e/kcp.go b/contrib/kcp/test/e2e/kcp.go index 31bc6ced4..3fa1dc2d5 100644 --- a/contrib/kcp/test/e2e/kcp.go +++ b/contrib/kcp/test/e2e/kcp.go @@ -18,25 +18,56 @@ package e2e import ( "fmt" + "path/filepath" "strings" "testing" "time" "github.com/kcp-dev/logicalcluster/v3" kcpapisv1alpha2 "github.com/kcp-dev/sdk/apis/apis/v1alpha2" + kcptenancyv1alpha1 "github.com/kcp-dev/sdk/apis/tenancy/v1alpha1" kcpclientset "github.com/kcp-dev/sdk/client/clientset/versioned/cluster" kcptestinghelpers "github.com/kcp-dev/sdk/testing/helpers" "github.com/spf13/pflag" "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/rest" "github.com/kube-bind/kube-bind/contrib/kcp/bootstrap" "github.com/kube-bind/kube-bind/contrib/kcp/bootstrap/options" + kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" "github.com/kube-bind/kube-bind/test/e2e/framework" ) +// waitForClusterEngaged blocks until the kcp multicluster provider has engaged +// the workspace described by cfg. The provider creates a kube-bind.io Cluster +// named "default" in every workspace it engages, so we poll for that object. +func waitForClusterEngaged(t testing.TB, cfg *rest.Config) { + t.Helper() + + gvr := schema.GroupVersionResource{ + Group: kubebindv1alpha2.SchemeGroupVersion.Group, + Version: kubebindv1alpha2.SchemeGroupVersion.Version, + Resource: "clusters", + } + + client := framework.DynamicClient(t, cfg).Resource(gvr) + + kcptestinghelpers.Eventually(t, func() (bool, string) { + _, err := client.Get(t.Context(), kubebindv1alpha2.DefaultClusterName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false, "kube-bind.io Cluster \"default\" not created yet" + } + return false, fmt.Sprintf("error getting kube-bind.io Cluster: %v", err) + } + return true, "" + }, wait.ForeverTestTimeout, time.Millisecond*100) +} + func wsConfig(t testing.TB, rest *rest.Config, workspace logicalcluster.Path) (*rest.Config, string) { cfg := rest cfg.Host += "/clusters/" + workspace.String() @@ -126,20 +157,46 @@ func createApiBinding(t testing.TB, client *kcpclientset.ClusterClientset, path return inCluster } -func bootstrapKCP(t testing.TB, rest *rest.Config) { +// bootstrapKCP bootstraps kcp for a test. It creates a per-test root workspace +// (root:) with a kbind workspace beneath it (root::kbind) and installs +// the kube-bind APIs there. It returns a rest.Config pointing at the per-test root +// workspace (so consumer/provider workspaces can be created as siblings of kbind) +// and the path of the kbind workspace. +func bootstrapKCP(t testing.TB, rest *rest.Config) (*rest.Config, logicalcluster.Path) { t.Helper() t.Log("Bootstrapping kcp") - cfg := rest - cfg.Host += "/clusters/root" - adminApiCfg := framework.RestToKubeconfig(cfg, "default") - adminKubeconfig := framework.WriteKubeconfig(t, adminApiCfg, "admin.kubeconfig") + wsName := strings.ToLower(t.Name()) + kbindLeafName := "kbind" + rootWorkspace := fmt.Sprintf("root:%s", wsName) + kbindWorkspace := fmt.Sprintf("%s:%s", rootWorkspace, kbindLeafName) + + // The provided rest.Config should already point to a running kcp server. + // It should already have /clusters/root in the path if read from kubeconfig. + // Just ensure we have the right base URL for bootstrap. + cfg := *rest + adminApiCfg := framework.RestToKubeconfig(&cfg, "default") + + // create the two workspaces we want our tests to run in + testRootCfg, _ := framework.NewWorkspace(t, rest, func(ws *kcptenancyv1alpha1.Workspace) { + ws.Name = wsName + ws.GenerateName = "" + }) + + framework.NewWorkspace(t, testRootCfg, func(ws *kcptenancyv1alpha1.Workspace) { + ws.Name = kbindLeafName + ws.GenerateName = "" + }) + + adminKubeconfig := framework.WriteKubeconfigToPath(t, adminApiCfg, filepath.Join(framework.WorkDir(t), "admin.kubeconfig")) fs := pflag.NewFlagSet("kcp-bootstrap", pflag.ContinueOnError) options := options.NewOptions() options.AddFlags(fs) options.KCPKubeConfig = adminKubeconfig + options.RootWorkspace = rootWorkspace + options.KBindWorkspace = kbindWorkspace completed, err := options.Complete() require.NoError(t, err) @@ -149,5 +206,9 @@ func bootstrapKCP(t testing.TB, rest *rest.Config) { bootstrapper, err := bootstrap.NewServer(t.Context(), config) require.NoError(t, err) - require.NoError(t, bootstrapper.Start(t.Context())) + + err = bootstrapper.Start(t.Context()) + require.NoError(t, err) + + return testRootCfg, logicalcluster.NewPath(kbindWorkspace) } diff --git a/contrib/kcp/test/e2e/kcp_3tier_test.go b/contrib/kcp/test/e2e/kcp_3tier_test.go index 06bbde170..fbead67a5 100644 --- a/contrib/kcp/test/e2e/kcp_3tier_test.go +++ b/contrib/kcp/test/e2e/kcp_3tier_test.go @@ -23,7 +23,6 @@ import ( "testing" "time" - "github.com/kcp-dev/logicalcluster/v3" kcpclientset "github.com/kcp-dev/sdk/client/clientset/versioned/cluster" kcptestinghelpers "github.com/kcp-dev/sdk/testing/helpers" "github.com/stretchr/testify/require" @@ -41,11 +40,13 @@ import ( // TestKCP3TierClusterScope tests the 3-tier kcp flow with cluster-scoped resources. func TestKCP3TierClusterScope(t *testing.T) { + t.Parallel() testKcp3TierIntegration(t, "3cc", kubebindv1alpha2.ClusterScope) } // TestKCP3TierNamespacedScope tests the 3-tier kcp flow with namespaced resources. func TestKCP3TierNamespacedScope(t *testing.T) { + t.Parallel() testKcp3TierIntegration(t, "3nc", kubebindv1alpha2.NamespacedScope) } @@ -68,7 +69,8 @@ func testKcp3TierIntegration(t *testing.T, name string, scope kubebindv1alpha2.I // kcp bootstrap (creates :root:kube-bind workspace with APIExport etc.) // Note: no StartDex needed — the backend uses embedded OIDC (mockoidc) which auto-approves. - bootstrapKCP(t, framework.ClientConfig(t)) + testRootCfg, kbindWsPath := bootstrapKCP(t, framework.ClientConfig(t)) + testRootPath, _ := kbindWsPath.Parent() suffix := framework.RandomString(4) @@ -80,8 +82,8 @@ func testKcp3TierIntegration(t *testing.T, name string, scope kubebindv1alpha2.I // --- Provider workspace --- t.Log("Create provider workspace") providerWsName := fmt.Sprintf("%s-provider-%s", name, suffix) - providerWsPath := logicalcluster.NewPath("root").Join(providerWsName) - providerCfg, _ := framework.NewWorkspace(t, framework.ClientConfig(t), framework.WithStaticName(providerWsName)) + providerWsPath := testRootPath.Join(providerWsName) + providerCfg, _ := framework.NewWorkspace(t, testRootCfg, framework.WithStaticName(providerWsName)) t.Log("Applying APIResourceSchema and APIExport to provider workspace") // Apply the cowboys APIResourceSchema and a cowboys-only APIExport. @@ -102,13 +104,13 @@ func testKcp3TierIntegration(t *testing.T, name string, scope kubebindv1alpha2.I // --- Consumer workspace --- t.Log("Create consumer workspace") consumerWsName := fmt.Sprintf("%s-consumer-%s", name, suffix) - consumerCfg, consumerKubeconfigPath := framework.NewWorkspace(t, framework.ClientConfig(t), framework.WithStaticName(consumerWsName)) + consumerCfg, consumerKubeconfigPath := framework.NewWorkspace(t, testRootCfg, framework.WithStaticName(consumerWsName)) t.Log("Start konnector for consumer workspace") framework.StartKonnector(t, consumerCfg, "--kubeconfig="+consumerKubeconfigPath) // --- Backend (kube-bind process + backend workspace) --- - backendAddr := bootstrapBackend(t, framework.ClientConfig(t), scope) + backendAddr := bootstrapBackend(t, framework.ClientConfig(t), scope, kbindWsPath) // --- Backend workspace --- // The backend workspace binds both kube-bind.io and the provider's cowboys APIExport. @@ -117,14 +119,21 @@ func testKcp3TierIntegration(t *testing.T, name string, scope kubebindv1alpha2.I // 2. Copy APIResourceSchemas with kube-bind.io/exported=true label t.Log("Create backend workspace") backendWsName := fmt.Sprintf("%s-backend-%s", name, suffix) - backendWsPath := logicalcluster.NewPath("root").Join(backendWsName) - backendCfg, _ := framework.NewWorkspace(t, framework.ClientConfig(t), framework.WithStaticName(backendWsName)) + backendWsPath := testRootPath.Join(backendWsName) + backendCfg, _ := framework.NewWorkspace(t, testRootCfg, framework.WithStaticName(backendWsName)) + + // Grant the embedded OIDC user permission to bind kube-bind.io resources in + // the backend workspace. This is required for the backend's bind + // SubjectAccessReview during the login flow. + rbacData, err := kcpboostrapdeploy.Examples.ReadFile("examples/rbac-embedded-user.yaml") + require.NoError(t, err, "failed to read embedded user RBAC") + framework.ApplyManifest(t, backendCfg, rbacData) t.Log("Bind kube-bind.io APIExport to backend workspace") createApiBinding(t, kcpClusterClient, backendWsPath, - generateApiBinding(t, logicalcluster.NewPath("root").Join("kube-bind"), "kube-bind.io", "kube-bind.io", + generateApiBinding(t, kbindWsPath, "kube-bind.io", "kube-bind.io", "clusterrolebindings.rbac.authorization.k8s.io", "clusterroles.rbac.authorization.k8s.io", "customresourcedefinitions.apiextensions.k8s.io", @@ -189,6 +198,11 @@ func testKcp3TierIntegration(t *testing.T, name string, scope kubebindv1alpha2.I backendClusterID := backendClusterSplit[len(backendClusterSplit)-1] require.NotEmpty(t, backendClusterID, "Retrieved cluster id is empty") + // Wait for the backend to engage the backend workspace before logging in, + // as the bind SubjectAccessReview during login depends on it. + t.Log("Wait for backend to engage backend workspace") + waitForClusterEngaged(t, backendCfg) + // --- Binding --- var templateRef, kind, resource string switch scope { @@ -204,7 +218,7 @@ func testKcp3TierIntegration(t *testing.T, name string, scope kubebindv1alpha2.I require.Fail(t, "unhandled scope %q", scope) } - kubeBindConfig := path.Join(framework.WorkDir, "kube-bind-config-kcp-3tier.yaml") + kubeBindConfig := path.Join(framework.WorkDir(t), "kube-bind-config-kcp-3tier.yaml") iostreams, _, _, _ := genericclioptions.NewTestIOStreams() authURLDryRunCh := make(chan string, 1) diff --git a/contrib/kcp/test/e2e/kcp_test.go b/contrib/kcp/test/e2e/kcp_test.go index 686c50e68..c9b9aa0d9 100644 --- a/contrib/kcp/test/e2e/kcp_test.go +++ b/contrib/kcp/test/e2e/kcp_test.go @@ -23,7 +23,6 @@ import ( "testing" "time" - "github.com/kcp-dev/logicalcluster/v3" kcpclientset "github.com/kcp-dev/sdk/client/clientset/versioned/cluster" kcptestinghelpers "github.com/kcp-dev/sdk/testing/helpers" "github.com/stretchr/testify/require" @@ -41,17 +40,13 @@ import ( "github.com/kube-bind/kube-bind/test/e2e/framework" ) -// TODO: Parallelizm is disabled due to bind-login overlapping server usage -// We need to refactor config machienery to allow multiple servers to be used in parallel tests -// https://github.com/kube-bind/kube-bind/issues/361 - func TestKCPClusterScope(t *testing.T) { - // t.Parallel() + t.Parallel() testKcpIntegration(t, "cc", kubebindv1alpha2.ClusterScope) } func TestKCPNamespacedScope(t *testing.T) { - // t.Parallel() + t.Parallel() testKcpIntegration(t, "nc", kubebindv1alpha2.NamespacedScope) } @@ -60,26 +55,30 @@ func testKcpIntegration(t *testing.T, name string, scope kubebindv1alpha2.Inform t.Logf("Testing kcp integration with informer scope %s, tempdir: %s", scope, t.TempDir()) // kcp bootstrap - bootstrapKCP(t, framework.ClientConfig(t)) + testRootCfg, kbindWsPath := bootstrapKCP(t, framework.ClientConfig(t)) + + // The per-test root workspace is the parent of the kbind workspace, i.e. + // kbindWsPath is root::kbind, so its parent is root:. + testRootPath, _ := kbindWsPath.Parent() suffix := framework.RandomString(4) // consumer t.Log("Create consumer workspace") consumerWsName := fmt.Sprintf("%s-consumer-%s", name, suffix) - consumerCfg, consumerKubeconfigPath := framework.NewWorkspace(t, framework.ClientConfig(t), framework.WithStaticName(consumerWsName)) + consumerCfg, consumerKubeconfigPath := framework.NewWorkspace(t, testRootCfg, framework.WithStaticName(consumerWsName)) t.Log("Start konnector for consumer workspace") framework.StartKonnector(t, consumerCfg, "--kubeconfig="+consumerKubeconfigPath) // backend - backendAddr := bootstrapBackend(t, framework.ClientConfig(t), scope) + backendAddr := bootstrapBackend(t, framework.ClientConfig(t), scope, kbindWsPath) // provider t.Log("Create provider workspace") providerWsName := fmt.Sprintf("%s-provider-%s", name, suffix) - providerWsPath := logicalcluster.NewPath("root").Join(providerWsName) - providerCfg, _ := framework.NewWorkspace(t, framework.ClientConfig(t), framework.WithStaticName(providerWsName)) + providerWsPath := testRootPath.Join(providerWsName) + providerCfg, _ := framework.NewWorkspace(t, testRootCfg, framework.WithStaticName(providerWsName)) cfg := framework.ClientConfig(t) cfg.Host = strings.Split(cfg.Host, "/clusters/")[0] @@ -90,7 +89,7 @@ func testKcpIntegration(t *testing.T, name string, scope kubebindv1alpha2.Inform createApiBinding(t, kcpClusterClient, providerWsPath, - generateApiBinding(t, logicalcluster.NewPath("root").Join("kube-bind"), "kube-bind.io", "kube-bind.io", + generateApiBinding(t, kbindWsPath, "kube-bind.io", "kube-bind.io", "clusterrolebindings.rbac.authorization.k8s.io", "clusterroles.rbac.authorization.k8s.io", "customresourcedefinitions.apiextensions.k8s.io", @@ -107,6 +106,13 @@ func testKcpIntegration(t *testing.T, name string, scope kubebindv1alpha2.Inform t.Log("Applying example APIExport, APIResourceSchemas and templates to provider workspace") + // Grant the embedded OIDC user permission to bind kube-bind.io resources in + // the provider workspace. This is required for the backend's bind + // SubjectAccessReview during the login flow. + rbacData, err := kcpboostrapdeploy.Examples.ReadFile("examples/rbac-embedded-user.yaml") + require.NoError(t, err, "failed to read embedded user RBAC") + framework.ApplyManifest(t, providerCfg, rbacData) + files := []string{"examples/apiexport.yaml", "examples/apiresourceschema-cowboys.yaml", // namespaced "examples/apiresourceschema-sheriffs.yaml", // cluster scoped @@ -155,6 +161,13 @@ func testKcpIntegration(t *testing.T, name string, scope kubebindv1alpha2.Inform providerClusterID := providerClusterSplit[len(providerClusterSplit)-1] require.NotEmpty(t, providerClusterID, "Retrieved cluster id is empty, source URL: %s", providerCluster.Status.URL) + // Wait for the backend to engage the provider workspace. The kcp provider + // creates a kube-bind.io Cluster named "default" in every workspace it + // engages, so its presence signals that the backend can serve requests for + // this cluster (which the bind SubjectAccessReview during login depends on). + t.Log("Wait for backend to engage provider workspace") + waitForClusterEngaged(t, providerCfg) + // kube-bind process t.Log("Perform binding process with browser") var templateRef, kind, resource string @@ -171,7 +184,7 @@ func testKcpIntegration(t *testing.T, name string, scope kubebindv1alpha2.Inform require.Fail(t, "unhandled scope %q", scope) } - kubeBindConfig := path.Join(framework.WorkDir, "kube-bind-config-kcp.yaml") + kubeBindConfig := path.Join(framework.WorkDir(t), "kube-bind-config-kcp.yaml") iostreams, _, _, _ := genericclioptions.NewTestIOStreams() authURLDryRunCh := make(chan string, 1) diff --git a/test/e2e/bind/happy-case_test.go b/test/e2e/bind/happy-case_test.go index 88bbe675d..df6c8f68f 100644 --- a/test/e2e/bind/happy-case_test.go +++ b/test/e2e/bind/happy-case_test.go @@ -249,7 +249,7 @@ func testHappyCase( providerCoreClient := framework.KubeClient(t, providerConfig).CoreV1() providerBindClient := framework.BindClient(t, providerConfig) - kubeBindConfig := path.Join(framework.WorkDir, "kube-bind-config.yaml") + kubeBindConfig := path.Join(framework.WorkDir(t), "kube-bind-config.yaml") // Secrets from the actual examples: // For cowboys: colt-45-permit (referenced), cowboy-gang-affiliation (label selector) diff --git a/test/e2e/framework/dex.go b/test/e2e/framework/dex.go index 6ef432d98..f14baedb3 100644 --- a/test/e2e/framework/dex.go +++ b/test/e2e/framework/dex.go @@ -56,7 +56,7 @@ func StartDex(t testing.TB) { dexOnce.Do(func() { dexConfig := os.Getenv("DEX_CONFIG") if dexConfig == "" { - dexConfig = filepath.Clean(filepath.Join(WorkDir, "..", "hack", "dex-config-dev.yaml")) + dexConfig = filepath.Clean(filepath.Join(RepoRoot(), "hack", "dex-config-dev.yaml")) } t.Logf("Starting dex with config %q", dexConfig) diff --git a/test/e2e/framework/env.go b/test/e2e/framework/env.go index 8881ac449..3864dc41e 100644 --- a/test/e2e/framework/env.go +++ b/test/e2e/framework/env.go @@ -20,15 +20,40 @@ import ( "os" "path/filepath" "runtime" + "strings" + "testing" ) var ( - WorkDir = os.Getenv("WORK_DIR") + repoRoot string ) func init() { - if WorkDir == "" { - _, f, _, _ := runtime.Caller(0) - WorkDir = filepath.Clean(filepath.Join(f, "..", "..", "..", "..", ".kube-bind")) + _, f, _, _ := runtime.Caller(0) + repoRoot = filepath.Clean(filepath.Join(f, "..", "..", "..", "..")) +} + +// WorkDir returns a test-specific working directory. Each test gets its own subdirectory +// based on the test name, allowing parallel tests to avoid file conflicts. +func WorkDir(t testing.TB) string { + workDir := os.Getenv("WORK_DIR") + if workDir == "" { + workDir = filepath.Join(RepoRoot(), ".kube-bind") } + + return filepath.Join(workDir, sanitizeTestName(t.Name())) +} + +// RepoRoot returns the root directory of the repository +func RepoRoot() string { + return repoRoot +} + +// sanitizeTestName converts a test name to a valid directory name by replacing +// path separators and other problematic characters. +func sanitizeTestName(name string) string { + name = strings.ReplaceAll(name, "/", "_") + name = strings.ReplaceAll(name, "\\", "_") + + return name } diff --git a/test/e2e/framework/kcp.go b/test/e2e/framework/kcp.go index b177b589d..9a5e04906 100644 --- a/test/e2e/framework/kcp.go +++ b/test/e2e/framework/kcp.go @@ -98,7 +98,7 @@ func WithGenerateName(s string, formatArgs ...any) ClusterWorkspaceOption { } } -func NewWorkspace(t *testing.T, config *rest.Config, options ...ClusterWorkspaceOption) (*rest.Config, string) { +func NewWorkspace(t testing.TB, config *rest.Config, options ...ClusterWorkspaceOption) (*rest.Config, string) { ctx, cancelFunc := context.WithCancel(context.Background()) t.Cleanup(cancelFunc) @@ -138,7 +138,7 @@ func NewWorkspace(t *testing.T, config *rest.Config, options ...ClusterWorkspace ret := rest.CopyConfig(config) ret.Host += ":" + ws.Name - wsKubeconfigPath := filepath.Join(WorkDir, ws.Name+".kubeconfig") + wsKubeconfigPath := filepath.Join(WorkDir(t), ws.Name+".kubeconfig") err = clientcmd.WriteToFile(RestToKubeconfig(ret, ""), wsKubeconfigPath) require.NoError(t, err) diff --git a/test/e2e/framework/kubeconfig.go b/test/e2e/framework/kubeconfig.go index f555bbc47..f369f71e3 100644 --- a/test/e2e/framework/kubeconfig.go +++ b/test/e2e/framework/kubeconfig.go @@ -65,8 +65,13 @@ func RestToKubeconfig(config *rest.Config, namespace string) clientcmdapi.Config func WriteKubeconfig(t testing.TB, config clientcmdapi.Config, name string) string { t.Helper() - p := filepath.Join(t.TempDir(), name) - err := clientcmd.WriteToFile(config, p) - require.NoError(t, err, "error writing kubeconfig to %q", p) - return p + + return WriteKubeconfigToPath(t, config, filepath.Join(t.TempDir(), name)) +} + +func WriteKubeconfigToPath(t testing.TB, config clientcmdapi.Config, path string) string { + t.Helper() + err := clientcmd.WriteToFile(config, path) + require.NoError(t, err, "error writing kubeconfig to %q", path) + return path }