diff --git a/contrib/kcp/bootstrap/config.go b/contrib/kcp/bootstrap/config.go index cdb874344..710066984 100644 --- a/contrib/kcp/bootstrap/config.go +++ b/contrib/kcp/bootstrap/config.go @@ -53,7 +53,7 @@ func NewConfig(options *options.CompletedOptions) (*Config, error) { return nil, err } config.ClientConfig = rest.CopyConfig(config.ClientConfig) - config.ClientConfig = rest.AddUserAgent(config.ClientConfig, "kube-bind-kcp-init") + config.ClientConfig = rest.AddUserAgent(config.ClientConfig, "kbind-kcp-init") config.ClientConfig, err = newKCPRestConfig(config.ClientConfig) if err != nil { diff --git a/contrib/kcp/bootstrap/config/config/bootstrap.go b/contrib/kcp/bootstrap/config/config/bootstrap.go index aa1639a4b..f22c614fd 100644 --- a/contrib/kcp/bootstrap/config/config/bootstrap.go +++ b/contrib/kcp/bootstrap/config/config/bootstrap.go @@ -24,7 +24,6 @@ import ( kcpdynamic "github.com/kcp-dev/client-go/dynamic" confighelpers "github.com/kcp-dev/kcp/config/helpers" "github.com/kcp-dev/logicalcluster/v3" - "github.com/kcp-dev/sdk/apis/core" kcpclientset "github.com/kcp-dev/sdk/client/clientset/versioned/cluster" "k8s.io/apimachinery/pkg/util/sets" ) @@ -32,9 +31,6 @@ import ( //go:embed *.yaml var fs embed.FS -// RootClusterName is the workspace to host common faros APIs. -var RootClusterName = logicalcluster.NewPath("root") - // Bootstrap creates resources in this package by continuously retrying the list. // This is blocking, i.e. it only returns (with error) when the context is closed or with nil when // the bootstrapping is successfully completed. @@ -44,8 +40,9 @@ func Bootstrap( apiExtensionClusterClient kcpapiextensionsclientset.ClusterInterface, dynamicClusterClient kcpdynamic.ClusterInterface, batteriesIncluded sets.Set[string], + baseWorkspace logicalcluster.Path, ) error { - rootDiscoveryClient := apiExtensionClusterClient.Cluster(core.RootCluster.Path()).Discovery() - rootDynamicClient := dynamicClusterClient.Cluster(core.RootCluster.Path()) - return confighelpers.Bootstrap(ctx, rootDiscoveryClient, rootDynamicClient, batteriesIncluded, fs) + discoveryClient := apiExtensionClusterClient.Cluster(baseWorkspace).Discovery() + dynamicClient := dynamicClusterClient.Cluster(baseWorkspace) + return confighelpers.Bootstrap(ctx, discoveryClient, dynamicClient, batteriesIncluded, fs) } diff --git a/contrib/kcp/bootstrap/config/config/clusterworkspace-kube-bind.yaml b/contrib/kcp/bootstrap/config/config/clusterworkspace-kube-bind.yaml index 0212e1112..0cbdb7733 100644 --- a/contrib/kcp/bootstrap/config/config/clusterworkspace-kube-bind.yaml +++ b/contrib/kcp/bootstrap/config/config/clusterworkspace-kube-bind.yaml @@ -6,5 +6,5 @@ metadata: bootstrap.kcp.io/create-only: "true" spec: type: - name: organization + name: universal path: root diff --git a/contrib/kcp/bootstrap/config/core/bootstrap.go b/contrib/kcp/bootstrap/config/core/bootstrap.go index ec42d3806..6de3f5805 100644 --- a/contrib/kcp/bootstrap/config/core/bootstrap.go +++ b/contrib/kcp/bootstrap/config/core/bootstrap.go @@ -28,11 +28,6 @@ import ( "github.com/kube-bind/kube-bind/contrib/kcp/bootstrap/config/core/resources" ) -var ( - // KubeBindRootClusterName is the workspace to host common APIs. - KubeBindRootClusterName = logicalcluster.NewPath("root:kube-bind") -) - // Bootstrap creates resources in this package by continuously retrying the list. // This is blocking, i.e. it only returns (with error) when the context is closed or with nil when // the bootstrapping is successfully completed. @@ -42,10 +37,11 @@ func Bootstrap( apiExtensionClusterClient kcpapiextensionsclientset.ClusterInterface, dynamicClusterClient kcpdynamic.ClusterInterface, batteriesIncluded sets.Set[string], + baseWorkspace logicalcluster.Path, ) error { - computeDiscoveryClient := apiExtensionClusterClient.Cluster(KubeBindRootClusterName).Discovery() - computeDynamicClient := dynamicClusterClient.Cluster(KubeBindRootClusterName) + discoveryClient := apiExtensionClusterClient.Cluster(baseWorkspace).Discovery() + dynamicClient := dynamicClusterClient.Cluster(baseWorkspace) - crdClient := apiExtensionClusterClient.ApiextensionsV1().Cluster(KubeBindRootClusterName).CustomResourceDefinitions() - return resources.Bootstrap(ctx, kcpClientSet, computeDiscoveryClient, computeDynamicClient, crdClient, batteriesIncluded) + crdClient := apiExtensionClusterClient.ApiextensionsV1().Cluster(baseWorkspace).CustomResourceDefinitions() + return resources.Bootstrap(ctx, kcpClientSet, discoveryClient, dynamicClient, crdClient, batteriesIncluded) } diff --git a/contrib/kcp/bootstrap/options/options.go b/contrib/kcp/bootstrap/options/options.go index 95870c331..1e9c75607 100644 --- a/contrib/kcp/bootstrap/options/options.go +++ b/contrib/kcp/bootstrap/options/options.go @@ -19,6 +19,7 @@ package options import ( "fmt" + "github.com/kcp-dev/sdk/apis/core" "github.com/spf13/pflag" "k8s.io/component-base/logs" logsv1 "k8s.io/component-base/logs/api/v1" @@ -29,9 +30,13 @@ type Options struct { ExtraOptions } + type ExtraOptions struct { KCPKubeConfig string KCPContext string + + RootWorkspace string + KBindWorkspace string } type completedOptions struct { @@ -50,8 +55,11 @@ func NewOptions() *Options { logs.Verbosity = logsv1.VerbosityLevel(2) return &Options{ - Logs: logs, - ExtraOptions: ExtraOptions{}, + Logs: logs, + ExtraOptions: ExtraOptions{ + RootWorkspace: core.RootCluster.Path().String(), + KBindWorkspace: "root:kbind", + }, } } @@ -60,6 +68,8 @@ func (options *Options) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&options.KCPKubeConfig, "kcp-kubeconfig", options.KCPKubeConfig, "path to a kcp kubeconfig. Required to bootstrap the server.") fs.StringVar(&options.KCPContext, "context", options.KCPContext, "Name of the context in the kcp kubeconfig file to use") + fs.StringVar(&options.RootWorkspace, "root-workspace", options.RootWorkspace, "Workspace path to use as the root (must exist already)") + fs.StringVar(&options.KBindWorkspace, "kbind-workspace", options.KBindWorkspace, "kbind workspace to use (must exist already)") } func (options *Options) Complete() (*CompletedOptions, error) { diff --git a/contrib/kcp/bootstrap/server.go b/contrib/kcp/bootstrap/server.go index 773d5a5a1..c50c68fe3 100644 --- a/contrib/kcp/bootstrap/server.go +++ b/contrib/kcp/bootstrap/server.go @@ -19,6 +19,7 @@ package bootstrap import ( "context" + "github.com/kcp-dev/logicalcluster/v3" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" @@ -49,17 +50,21 @@ func (s *Server) Start(ctx context.Context) error { s.Config.ApiextensionsClient, s.Config.DynamicClusterClient, fakeBatteries, + logicalcluster.NewPath(s.Config.Options.ExtraOptions.RootWorkspace), ); err != nil { logger.Error(err, "failed to bootstrap initial config workspace") return nil // don't klog.Fatal. This only happens when context is cancelled. } + kbindWorkspace := logicalcluster.NewPath(s.Config.Options.ExtraOptions.KBindWorkspace) + if err := bootstrapcore.Bootstrap( ctx, s.Config.KcpClusterClient, s.Config.ApiextensionsClient, s.Config.DynamicClusterClient, fakeBatteries, + kbindWorkspace, ); err != nil { logger.Error(err, "failed to bootstrap core workspace") return nil // don't klog.Fatal. This only happens when context is cancelled. @@ -71,6 +76,7 @@ func (s *Server) Start(ctx context.Context) error { s.Config.ApiextensionsClient, s.Config.DynamicClusterClient, fakeBatteries, + kbindWorkspace, ); err != nil { logger.Error(err, "failed to bootstrap workspace") return nil // don't klog.Fatal. This only happens when context is cancelled. diff --git a/contrib/kcp/deploy/bootstrap.go b/contrib/kcp/deploy/bootstrap.go index 5808e12ad..af499bd65 100644 --- a/contrib/kcp/deploy/bootstrap.go +++ b/contrib/kcp/deploy/bootstrap.go @@ -39,11 +39,6 @@ import ( //go:embed examples/*.yaml var Examples embed.FS -var ( - // KubeBindRootClusterName is the workspace to host common APIs. - KubeBindRootClusterName = logicalcluster.NewPath("root:kube-bind") -) - // Bootstrap creates resources in this package by continuously retrying the list. // This is blocking, i.e. it only returns (with error) when the context is closed or with nil when // the bootstrapping is successfully completed. @@ -53,21 +48,22 @@ func Bootstrap( apiExtensionClusterClient kcpapiextensionsclientset.ClusterInterface, dynamicClusterClient kcpdynamic.ClusterInterface, batteriesIncluded sets.Set[string], + baseWorkspace logicalcluster.Path, ) error { - computeDiscoveryClient := apiExtensionClusterClient.Cluster(KubeBindRootClusterName).Discovery() - computeDynamicClient := dynamicClusterClient.Cluster(KubeBindRootClusterName) + discoveryClient := apiExtensionClusterClient.Cluster(baseWorkspace).Discovery() + dynamicClient := dynamicClusterClient.Cluster(baseWorkspace) - crdClient := apiExtensionClusterClient.ApiextensionsV1().Cluster(KubeBindRootClusterName).CustomResourceDefinitions() - kcpClient := kcpClientSet.Cluster(KubeBindRootClusterName) + crdClient := apiExtensionClusterClient.ApiextensionsV1().Cluster(baseWorkspace).CustomResourceDefinitions() + kcpClient := kcpClientSet.Cluster(baseWorkspace) - err := resources.Bootstrap(ctx, kcpClientSet, computeDiscoveryClient, computeDynamicClient, crdClient, batteriesIncluded) + err := resources.Bootstrap(ctx, kcpClientSet, discoveryClient, dynamicClient, crdClient, batteriesIncluded) if err != nil { return err } // create recursive apibinding so we can start controllers. // this is a temporary solution until we have a better way to bootstrap controllers. - return bindAPIExport(ctx, kcpClient, "kube-bind.io", KubeBindRootClusterName) + return bindAPIExport(ctx, kcpClient, "kube-bind.io", baseWorkspace) } func bindAPIExport(ctx context.Context, kcpClient kcpclient.Interface, exportName string, clusterPath logicalcluster.Path) error { diff --git a/contrib/kcp/test/e2e/backend.go b/contrib/kcp/test/e2e/backend.go index ff033191c..8306651ef 100644 --- a/contrib/kcp/test/e2e/backend.go +++ b/contrib/kcp/test/e2e/backend.go @@ -35,7 +35,7 @@ import ( "github.com/kube-bind/kube-bind/test/e2e/framework" ) -func bootstrapBackend(t *testing.T, rest *rest.Config, scope kubebindv1alpha2.InformerScope) string { +func bootstrapBackend(t *testing.T, rest *rest.Config, scope kubebindv1alpha2.InformerScope, kbindPath logicalcluster.Path) string { t.Helper() t.Log("Bootstrapping backend") @@ -46,7 +46,7 @@ func bootstrapBackend(t *testing.T, rest *rest.Config, scope kubebindv1alpha2.In exportUrl := "" kcptestinghelpers.Eventually(t, func() (bool, string) { - exportES, err := client.Cluster(logicalcluster.NewPath("root").Join("kube-bind")). + exportES, err := client.Cluster(kbindPath). ApisV1alpha1(). APIExportEndpointSlices(). Get(t.Context(), "kube-bind.io", metav1.GetOptions{}) @@ -61,7 +61,7 @@ func bootstrapBackend(t *testing.T, rest *rest.Config, scope kubebindv1alpha2.In }, wait.ForeverTestTimeout, time.Millisecond*100) require.NotEmpty(t, exportUrl, "APIExportEndpointSlice URL is empty") - _, backendKubeconfig := wsConfig(t, rest, logicalcluster.NewPath("root").Join("kube-bind")) + _, backendKubeconfig := wsConfig(t, rest, kbindPath) t.Log("Starting kube-bind backend for kcp") addr, _ := framework.StartBackend(t, diff --git a/contrib/kcp/test/e2e/kcp.go b/contrib/kcp/test/e2e/kcp.go index 31bc6ced4..3fa1dc2d5 100644 --- a/contrib/kcp/test/e2e/kcp.go +++ b/contrib/kcp/test/e2e/kcp.go @@ -18,25 +18,56 @@ package e2e import ( "fmt" + "path/filepath" "strings" "testing" "time" "github.com/kcp-dev/logicalcluster/v3" kcpapisv1alpha2 "github.com/kcp-dev/sdk/apis/apis/v1alpha2" + kcptenancyv1alpha1 "github.com/kcp-dev/sdk/apis/tenancy/v1alpha1" kcpclientset "github.com/kcp-dev/sdk/client/clientset/versioned/cluster" kcptestinghelpers "github.com/kcp-dev/sdk/testing/helpers" "github.com/spf13/pflag" "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/rest" "github.com/kube-bind/kube-bind/contrib/kcp/bootstrap" "github.com/kube-bind/kube-bind/contrib/kcp/bootstrap/options" + kubebindv1alpha2 "github.com/kube-bind/kube-bind/sdk/apis/kubebind/v1alpha2" "github.com/kube-bind/kube-bind/test/e2e/framework" ) +// waitForClusterEngaged blocks until the kcp multicluster provider has engaged +// the workspace described by cfg. The provider creates a kube-bind.io Cluster +// named "default" in every workspace it engages, so we poll for that object. +func waitForClusterEngaged(t testing.TB, cfg *rest.Config) { + t.Helper() + + gvr := schema.GroupVersionResource{ + Group: kubebindv1alpha2.SchemeGroupVersion.Group, + Version: kubebindv1alpha2.SchemeGroupVersion.Version, + Resource: "clusters", + } + + client := framework.DynamicClient(t, cfg).Resource(gvr) + + kcptestinghelpers.Eventually(t, func() (bool, string) { + _, err := client.Get(t.Context(), kubebindv1alpha2.DefaultClusterName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false, "kube-bind.io Cluster \"default\" not created yet" + } + return false, fmt.Sprintf("error getting kube-bind.io Cluster: %v", err) + } + return true, "" + }, wait.ForeverTestTimeout, time.Millisecond*100) +} + func wsConfig(t testing.TB, rest *rest.Config, workspace logicalcluster.Path) (*rest.Config, string) { cfg := rest cfg.Host += "/clusters/" + workspace.String() @@ -126,20 +157,46 @@ func createApiBinding(t testing.TB, client *kcpclientset.ClusterClientset, path return inCluster } -func bootstrapKCP(t testing.TB, rest *rest.Config) { +// bootstrapKCP bootstraps kcp for a test. It creates a per-test root workspace +// (root:) with a kbind workspace beneath it (root::kbind) and installs +// the kube-bind APIs there. It returns a rest.Config pointing at the per-test root +// workspace (so consumer/provider workspaces can be created as siblings of kbind) +// and the path of the kbind workspace. +func bootstrapKCP(t testing.TB, rest *rest.Config) (*rest.Config, logicalcluster.Path) { t.Helper() t.Log("Bootstrapping kcp") - cfg := rest - cfg.Host += "/clusters/root" - adminApiCfg := framework.RestToKubeconfig(cfg, "default") - adminKubeconfig := framework.WriteKubeconfig(t, adminApiCfg, "admin.kubeconfig") + wsName := strings.ToLower(t.Name()) + kbindLeafName := "kbind" + rootWorkspace := fmt.Sprintf("root:%s", wsName) + kbindWorkspace := fmt.Sprintf("%s:%s", rootWorkspace, kbindLeafName) + + // The provided rest.Config should already point to a running kcp server. + // It should already have /clusters/root in the path if read from kubeconfig. + // Just ensure we have the right base URL for bootstrap. + cfg := *rest + adminApiCfg := framework.RestToKubeconfig(&cfg, "default") + + // create the two workspaces we want our tests to run in + testRootCfg, _ := framework.NewWorkspace(t, rest, func(ws *kcptenancyv1alpha1.Workspace) { + ws.Name = wsName + ws.GenerateName = "" + }) + + framework.NewWorkspace(t, testRootCfg, func(ws *kcptenancyv1alpha1.Workspace) { + ws.Name = kbindLeafName + ws.GenerateName = "" + }) + + adminKubeconfig := framework.WriteKubeconfigToPath(t, adminApiCfg, filepath.Join(framework.WorkDir(t), "admin.kubeconfig")) fs := pflag.NewFlagSet("kcp-bootstrap", pflag.ContinueOnError) options := options.NewOptions() options.AddFlags(fs) options.KCPKubeConfig = adminKubeconfig + options.RootWorkspace = rootWorkspace + options.KBindWorkspace = kbindWorkspace completed, err := options.Complete() require.NoError(t, err) @@ -149,5 +206,9 @@ func bootstrapKCP(t testing.TB, rest *rest.Config) { bootstrapper, err := bootstrap.NewServer(t.Context(), config) require.NoError(t, err) - require.NoError(t, bootstrapper.Start(t.Context())) + + err = bootstrapper.Start(t.Context()) + require.NoError(t, err) + + return testRootCfg, logicalcluster.NewPath(kbindWorkspace) } diff --git a/contrib/kcp/test/e2e/kcp_3tier_test.go b/contrib/kcp/test/e2e/kcp_3tier_test.go index 06bbde170..fbead67a5 100644 --- a/contrib/kcp/test/e2e/kcp_3tier_test.go +++ b/contrib/kcp/test/e2e/kcp_3tier_test.go @@ -23,7 +23,6 @@ import ( "testing" "time" - "github.com/kcp-dev/logicalcluster/v3" kcpclientset "github.com/kcp-dev/sdk/client/clientset/versioned/cluster" kcptestinghelpers "github.com/kcp-dev/sdk/testing/helpers" "github.com/stretchr/testify/require" @@ -41,11 +40,13 @@ import ( // TestKCP3TierClusterScope tests the 3-tier kcp flow with cluster-scoped resources. func TestKCP3TierClusterScope(t *testing.T) { + t.Parallel() testKcp3TierIntegration(t, "3cc", kubebindv1alpha2.ClusterScope) } // TestKCP3TierNamespacedScope tests the 3-tier kcp flow with namespaced resources. func TestKCP3TierNamespacedScope(t *testing.T) { + t.Parallel() testKcp3TierIntegration(t, "3nc", kubebindv1alpha2.NamespacedScope) } @@ -68,7 +69,8 @@ func testKcp3TierIntegration(t *testing.T, name string, scope kubebindv1alpha2.I // kcp bootstrap (creates :root:kube-bind workspace with APIExport etc.) // Note: no StartDex needed — the backend uses embedded OIDC (mockoidc) which auto-approves. - bootstrapKCP(t, framework.ClientConfig(t)) + testRootCfg, kbindWsPath := bootstrapKCP(t, framework.ClientConfig(t)) + testRootPath, _ := kbindWsPath.Parent() suffix := framework.RandomString(4) @@ -80,8 +82,8 @@ func testKcp3TierIntegration(t *testing.T, name string, scope kubebindv1alpha2.I // --- Provider workspace --- t.Log("Create provider workspace") providerWsName := fmt.Sprintf("%s-provider-%s", name, suffix) - providerWsPath := logicalcluster.NewPath("root").Join(providerWsName) - providerCfg, _ := framework.NewWorkspace(t, framework.ClientConfig(t), framework.WithStaticName(providerWsName)) + providerWsPath := testRootPath.Join(providerWsName) + providerCfg, _ := framework.NewWorkspace(t, testRootCfg, framework.WithStaticName(providerWsName)) t.Log("Applying APIResourceSchema and APIExport to provider workspace") // Apply the cowboys APIResourceSchema and a cowboys-only APIExport. @@ -102,13 +104,13 @@ func testKcp3TierIntegration(t *testing.T, name string, scope kubebindv1alpha2.I // --- Consumer workspace --- t.Log("Create consumer workspace") consumerWsName := fmt.Sprintf("%s-consumer-%s", name, suffix) - consumerCfg, consumerKubeconfigPath := framework.NewWorkspace(t, framework.ClientConfig(t), framework.WithStaticName(consumerWsName)) + consumerCfg, consumerKubeconfigPath := framework.NewWorkspace(t, testRootCfg, framework.WithStaticName(consumerWsName)) t.Log("Start konnector for consumer workspace") framework.StartKonnector(t, consumerCfg, "--kubeconfig="+consumerKubeconfigPath) // --- Backend (kube-bind process + backend workspace) --- - backendAddr := bootstrapBackend(t, framework.ClientConfig(t), scope) + backendAddr := bootstrapBackend(t, framework.ClientConfig(t), scope, kbindWsPath) // --- Backend workspace --- // The backend workspace binds both kube-bind.io and the provider's cowboys APIExport. @@ -117,14 +119,21 @@ func testKcp3TierIntegration(t *testing.T, name string, scope kubebindv1alpha2.I // 2. Copy APIResourceSchemas with kube-bind.io/exported=true label t.Log("Create backend workspace") backendWsName := fmt.Sprintf("%s-backend-%s", name, suffix) - backendWsPath := logicalcluster.NewPath("root").Join(backendWsName) - backendCfg, _ := framework.NewWorkspace(t, framework.ClientConfig(t), framework.WithStaticName(backendWsName)) + backendWsPath := testRootPath.Join(backendWsName) + backendCfg, _ := framework.NewWorkspace(t, testRootCfg, framework.WithStaticName(backendWsName)) + + // Grant the embedded OIDC user permission to bind kube-bind.io resources in + // the backend workspace. This is required for the backend's bind + // SubjectAccessReview during the login flow. + rbacData, err := kcpboostrapdeploy.Examples.ReadFile("examples/rbac-embedded-user.yaml") + require.NoError(t, err, "failed to read embedded user RBAC") + framework.ApplyManifest(t, backendCfg, rbacData) t.Log("Bind kube-bind.io APIExport to backend workspace") createApiBinding(t, kcpClusterClient, backendWsPath, - generateApiBinding(t, logicalcluster.NewPath("root").Join("kube-bind"), "kube-bind.io", "kube-bind.io", + generateApiBinding(t, kbindWsPath, "kube-bind.io", "kube-bind.io", "clusterrolebindings.rbac.authorization.k8s.io", "clusterroles.rbac.authorization.k8s.io", "customresourcedefinitions.apiextensions.k8s.io", @@ -189,6 +198,11 @@ func testKcp3TierIntegration(t *testing.T, name string, scope kubebindv1alpha2.I backendClusterID := backendClusterSplit[len(backendClusterSplit)-1] require.NotEmpty(t, backendClusterID, "Retrieved cluster id is empty") + // Wait for the backend to engage the backend workspace before logging in, + // as the bind SubjectAccessReview during login depends on it. + t.Log("Wait for backend to engage backend workspace") + waitForClusterEngaged(t, backendCfg) + // --- Binding --- var templateRef, kind, resource string switch scope { @@ -204,7 +218,7 @@ func testKcp3TierIntegration(t *testing.T, name string, scope kubebindv1alpha2.I require.Fail(t, "unhandled scope %q", scope) } - kubeBindConfig := path.Join(framework.WorkDir, "kube-bind-config-kcp-3tier.yaml") + kubeBindConfig := path.Join(framework.WorkDir(t), "kube-bind-config-kcp-3tier.yaml") iostreams, _, _, _ := genericclioptions.NewTestIOStreams() authURLDryRunCh := make(chan string, 1) diff --git a/contrib/kcp/test/e2e/kcp_test.go b/contrib/kcp/test/e2e/kcp_test.go index 686c50e68..c9b9aa0d9 100644 --- a/contrib/kcp/test/e2e/kcp_test.go +++ b/contrib/kcp/test/e2e/kcp_test.go @@ -23,7 +23,6 @@ import ( "testing" "time" - "github.com/kcp-dev/logicalcluster/v3" kcpclientset "github.com/kcp-dev/sdk/client/clientset/versioned/cluster" kcptestinghelpers "github.com/kcp-dev/sdk/testing/helpers" "github.com/stretchr/testify/require" @@ -41,17 +40,13 @@ import ( "github.com/kube-bind/kube-bind/test/e2e/framework" ) -// TODO: Parallelizm is disabled due to bind-login overlapping server usage -// We need to refactor config machienery to allow multiple servers to be used in parallel tests -// https://github.com/kube-bind/kube-bind/issues/361 - func TestKCPClusterScope(t *testing.T) { - // t.Parallel() + t.Parallel() testKcpIntegration(t, "cc", kubebindv1alpha2.ClusterScope) } func TestKCPNamespacedScope(t *testing.T) { - // t.Parallel() + t.Parallel() testKcpIntegration(t, "nc", kubebindv1alpha2.NamespacedScope) } @@ -60,26 +55,30 @@ func testKcpIntegration(t *testing.T, name string, scope kubebindv1alpha2.Inform t.Logf("Testing kcp integration with informer scope %s, tempdir: %s", scope, t.TempDir()) // kcp bootstrap - bootstrapKCP(t, framework.ClientConfig(t)) + testRootCfg, kbindWsPath := bootstrapKCP(t, framework.ClientConfig(t)) + + // The per-test root workspace is the parent of the kbind workspace, i.e. + // kbindWsPath is root::kbind, so its parent is root:. + testRootPath, _ := kbindWsPath.Parent() suffix := framework.RandomString(4) // consumer t.Log("Create consumer workspace") consumerWsName := fmt.Sprintf("%s-consumer-%s", name, suffix) - consumerCfg, consumerKubeconfigPath := framework.NewWorkspace(t, framework.ClientConfig(t), framework.WithStaticName(consumerWsName)) + consumerCfg, consumerKubeconfigPath := framework.NewWorkspace(t, testRootCfg, framework.WithStaticName(consumerWsName)) t.Log("Start konnector for consumer workspace") framework.StartKonnector(t, consumerCfg, "--kubeconfig="+consumerKubeconfigPath) // backend - backendAddr := bootstrapBackend(t, framework.ClientConfig(t), scope) + backendAddr := bootstrapBackend(t, framework.ClientConfig(t), scope, kbindWsPath) // provider t.Log("Create provider workspace") providerWsName := fmt.Sprintf("%s-provider-%s", name, suffix) - providerWsPath := logicalcluster.NewPath("root").Join(providerWsName) - providerCfg, _ := framework.NewWorkspace(t, framework.ClientConfig(t), framework.WithStaticName(providerWsName)) + providerWsPath := testRootPath.Join(providerWsName) + providerCfg, _ := framework.NewWorkspace(t, testRootCfg, framework.WithStaticName(providerWsName)) cfg := framework.ClientConfig(t) cfg.Host = strings.Split(cfg.Host, "/clusters/")[0] @@ -90,7 +89,7 @@ func testKcpIntegration(t *testing.T, name string, scope kubebindv1alpha2.Inform createApiBinding(t, kcpClusterClient, providerWsPath, - generateApiBinding(t, logicalcluster.NewPath("root").Join("kube-bind"), "kube-bind.io", "kube-bind.io", + generateApiBinding(t, kbindWsPath, "kube-bind.io", "kube-bind.io", "clusterrolebindings.rbac.authorization.k8s.io", "clusterroles.rbac.authorization.k8s.io", "customresourcedefinitions.apiextensions.k8s.io", @@ -107,6 +106,13 @@ func testKcpIntegration(t *testing.T, name string, scope kubebindv1alpha2.Inform t.Log("Applying example APIExport, APIResourceSchemas and templates to provider workspace") + // Grant the embedded OIDC user permission to bind kube-bind.io resources in + // the provider workspace. This is required for the backend's bind + // SubjectAccessReview during the login flow. + rbacData, err := kcpboostrapdeploy.Examples.ReadFile("examples/rbac-embedded-user.yaml") + require.NoError(t, err, "failed to read embedded user RBAC") + framework.ApplyManifest(t, providerCfg, rbacData) + files := []string{"examples/apiexport.yaml", "examples/apiresourceschema-cowboys.yaml", // namespaced "examples/apiresourceschema-sheriffs.yaml", // cluster scoped @@ -155,6 +161,13 @@ func testKcpIntegration(t *testing.T, name string, scope kubebindv1alpha2.Inform providerClusterID := providerClusterSplit[len(providerClusterSplit)-1] require.NotEmpty(t, providerClusterID, "Retrieved cluster id is empty, source URL: %s", providerCluster.Status.URL) + // Wait for the backend to engage the provider workspace. The kcp provider + // creates a kube-bind.io Cluster named "default" in every workspace it + // engages, so its presence signals that the backend can serve requests for + // this cluster (which the bind SubjectAccessReview during login depends on). + t.Log("Wait for backend to engage provider workspace") + waitForClusterEngaged(t, providerCfg) + // kube-bind process t.Log("Perform binding process with browser") var templateRef, kind, resource string @@ -171,7 +184,7 @@ func testKcpIntegration(t *testing.T, name string, scope kubebindv1alpha2.Inform require.Fail(t, "unhandled scope %q", scope) } - kubeBindConfig := path.Join(framework.WorkDir, "kube-bind-config-kcp.yaml") + kubeBindConfig := path.Join(framework.WorkDir(t), "kube-bind-config-kcp.yaml") iostreams, _, _, _ := genericclioptions.NewTestIOStreams() authURLDryRunCh := make(chan string, 1) diff --git a/test/e2e/bind/happy-case_test.go b/test/e2e/bind/happy-case_test.go index 88bbe675d..df6c8f68f 100644 --- a/test/e2e/bind/happy-case_test.go +++ b/test/e2e/bind/happy-case_test.go @@ -249,7 +249,7 @@ func testHappyCase( providerCoreClient := framework.KubeClient(t, providerConfig).CoreV1() providerBindClient := framework.BindClient(t, providerConfig) - kubeBindConfig := path.Join(framework.WorkDir, "kube-bind-config.yaml") + kubeBindConfig := path.Join(framework.WorkDir(t), "kube-bind-config.yaml") // Secrets from the actual examples: // For cowboys: colt-45-permit (referenced), cowboy-gang-affiliation (label selector) diff --git a/test/e2e/framework/dex.go b/test/e2e/framework/dex.go index 6ef432d98..f14baedb3 100644 --- a/test/e2e/framework/dex.go +++ b/test/e2e/framework/dex.go @@ -56,7 +56,7 @@ func StartDex(t testing.TB) { dexOnce.Do(func() { dexConfig := os.Getenv("DEX_CONFIG") if dexConfig == "" { - dexConfig = filepath.Clean(filepath.Join(WorkDir, "..", "hack", "dex-config-dev.yaml")) + dexConfig = filepath.Clean(filepath.Join(RepoRoot(), "hack", "dex-config-dev.yaml")) } t.Logf("Starting dex with config %q", dexConfig) diff --git a/test/e2e/framework/env.go b/test/e2e/framework/env.go index 8881ac449..3864dc41e 100644 --- a/test/e2e/framework/env.go +++ b/test/e2e/framework/env.go @@ -20,15 +20,40 @@ import ( "os" "path/filepath" "runtime" + "strings" + "testing" ) var ( - WorkDir = os.Getenv("WORK_DIR") + repoRoot string ) func init() { - if WorkDir == "" { - _, f, _, _ := runtime.Caller(0) - WorkDir = filepath.Clean(filepath.Join(f, "..", "..", "..", "..", ".kube-bind")) + _, f, _, _ := runtime.Caller(0) + repoRoot = filepath.Clean(filepath.Join(f, "..", "..", "..", "..")) +} + +// WorkDir returns a test-specific working directory. Each test gets its own subdirectory +// based on the test name, allowing parallel tests to avoid file conflicts. +func WorkDir(t testing.TB) string { + workDir := os.Getenv("WORK_DIR") + if workDir == "" { + workDir = filepath.Join(RepoRoot(), ".kube-bind") } + + return filepath.Join(workDir, sanitizeTestName(t.Name())) +} + +// RepoRoot returns the root directory of the repository +func RepoRoot() string { + return repoRoot +} + +// sanitizeTestName converts a test name to a valid directory name by replacing +// path separators and other problematic characters. +func sanitizeTestName(name string) string { + name = strings.ReplaceAll(name, "/", "_") + name = strings.ReplaceAll(name, "\\", "_") + + return name } diff --git a/test/e2e/framework/kcp.go b/test/e2e/framework/kcp.go index b177b589d..9a5e04906 100644 --- a/test/e2e/framework/kcp.go +++ b/test/e2e/framework/kcp.go @@ -98,7 +98,7 @@ func WithGenerateName(s string, formatArgs ...any) ClusterWorkspaceOption { } } -func NewWorkspace(t *testing.T, config *rest.Config, options ...ClusterWorkspaceOption) (*rest.Config, string) { +func NewWorkspace(t testing.TB, config *rest.Config, options ...ClusterWorkspaceOption) (*rest.Config, string) { ctx, cancelFunc := context.WithCancel(context.Background()) t.Cleanup(cancelFunc) @@ -138,7 +138,7 @@ func NewWorkspace(t *testing.T, config *rest.Config, options ...ClusterWorkspace ret := rest.CopyConfig(config) ret.Host += ":" + ws.Name - wsKubeconfigPath := filepath.Join(WorkDir, ws.Name+".kubeconfig") + wsKubeconfigPath := filepath.Join(WorkDir(t), ws.Name+".kubeconfig") err = clientcmd.WriteToFile(RestToKubeconfig(ret, ""), wsKubeconfigPath) require.NoError(t, err) diff --git a/test/e2e/framework/kubeconfig.go b/test/e2e/framework/kubeconfig.go index f555bbc47..f369f71e3 100644 --- a/test/e2e/framework/kubeconfig.go +++ b/test/e2e/framework/kubeconfig.go @@ -65,8 +65,13 @@ func RestToKubeconfig(config *rest.Config, namespace string) clientcmdapi.Config func WriteKubeconfig(t testing.TB, config clientcmdapi.Config, name string) string { t.Helper() - p := filepath.Join(t.TempDir(), name) - err := clientcmd.WriteToFile(config, p) - require.NoError(t, err, "error writing kubeconfig to %q", p) - return p + + return WriteKubeconfigToPath(t, config, filepath.Join(t.TempDir(), name)) +} + +func WriteKubeconfigToPath(t testing.TB, config clientcmdapi.Config, path string) string { + t.Helper() + err := clientcmd.WriteToFile(config, path) + require.NoError(t, err, "error writing kubeconfig to %q", path) + return path }