diff --git a/Makefile b/Makefile index af3905b76d35..721cbdcfdb9a 100644 --- a/Makefile +++ b/Makefile @@ -443,7 +443,7 @@ test: generate test-e2ev2-unit .PHONY: test-e2ev2-unit test-e2ev2-unit: $(GO) test $(GO_TEST_FLAGS) -tags=e2ev2 -count=1 -timeout=10m ./test/e2e/v2/internal ./test/e2e/v2/cmd/run-tests - $(GO) test $(GO_TEST_FLAGS) -tags=e2ev2 -count=1 -timeout=10m -run '^(TestComponentSupportsTLSConfig|TestVerifyIntermediateTLSConnectivityAfterDowngrade)$$' ./test/e2e/v2/tests + $(GO) test $(GO_TEST_FLAGS) -tags=e2ev2 -count=1 -timeout=10m -run '^(TestComponentSupportsTLSConfig|TestVerifyIntermediateTLSConnectivityAfterDowngrade|TestImageMirroring.*)$$' ./test/e2e/v2/tests # Run tests only for Go packages with changes relative to PULL_BASE_SHA. # Skips entirely if no .go files changed. No generate dependency (verify-quick handles it). diff --git a/test/e2e/v2/tests/hosted_cluster_image_mirroring_helpers.go b/test/e2e/v2/tests/hosted_cluster_image_mirroring_helpers.go index 0d8a09886ea1..496f2168ebca 100644 --- a/test/e2e/v2/tests/hosted_cluster_image_mirroring_helpers.go +++ b/test/e2e/v2/tests/hosted_cluster_image_mirroring_helpers.go @@ -9,11 +9,13 @@ import ( "strings" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/test/e2e/v2/util/mirrorregistry" configv1 "github.com/openshift/api/config/v1" routev1 "github.com/openshift/api/route/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/sets" crclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -24,6 +26,11 @@ const ( imageMirroringIgnitionName = "ignition-config-40-image-content-source" imageMirroringWorkerLabel = "machineconfiguration.openshift.io/role" imageMirroringCoreLabel = "hypershift.openshift.io/core-ignition-config" + + // imageMirroringVirtualSource is the fixture source repository that only + // exists on the mirror: its ".invalid" host never resolves (RFC 2606), so a + // successful pull from it can only have been served by an IDMS mirror. + imageMirroringVirtualSource = "mirror-source.hypershift-e2e.invalid/libpod/registry" ) // imageMirroringConfigured reports whether the HC opted into image mirroring @@ -33,6 +40,29 @@ func imageMirroringConfigured(hc *hyperv1.HostedCluster, routeFound bool) bool { return hc != nil && len(hc.Spec.ImageContentSources) > 0 && routeFound } +// imageMirroringLoadSetup reads the mirror fixture Route from the management +// cluster and validates the HC image-mirroring setup against it. It returns +// configured=false and no error when the HC has no imageContentSources or the +// Route does not exist, so the caller can skip. A Route read error other than +// NotFound, or an invalid setup once both are present, is returned as an error +// so the caller fails instead of skipping. +func imageMirroringLoadSetup(ctx context.Context, client crclient.Client, hc *hyperv1.HostedCluster) (bool, []hyperv1.ImageContentSource, string, error) { + key := crclient.ObjectKey{Namespace: mirrorregistry.DefaultNamespace, Name: mirrorregistry.DefaultName} + route := &routev1.Route{} + err := client.Get(ctx, key, route) + if err != nil && !apierrors.IsNotFound(err) { + return false, nil, "", fmt.Errorf("read mirror fixture Route %s: %w", key, err) + } + if !imageMirroringConfigured(hc, err == nil) { + return false, nil, "", nil + } + expected, err := imageMirroringExpected(hc, route) + if err != nil { + return false, nil, "", err + } + return true, expected, route.Spec.Host, nil +} + // Expectations come only from a validated snapshot of HC input, never from a // producer conversion or either output. Mirror priority and multiplicity matter. func imageMirroringExpected(hc *hyperv1.HostedCluster, route *routev1.Route) ([]hyperv1.ImageContentSource, error) { @@ -58,7 +88,7 @@ func imageMirroringExpected(hc *hyperv1.HostedCluster, route *routev1.Route) ([] if route == nil || strings.TrimSpace(route.Spec.Host) == "" { return nil, fmt.Errorf("mirror fixture Route has an empty host") } - for _, source := range []string{"quay.io/libpod/registry", "mirror-source.hypershift-e2e.invalid/libpod/registry"} { + for _, source := range []string{"quay.io/libpod/registry", imageMirroringVirtualSource} { found := false for _, entry := range expected { if entry.Source != source { diff --git a/test/e2e/v2/tests/hosted_cluster_image_mirroring_helpers_test.go b/test/e2e/v2/tests/hosted_cluster_image_mirroring_helpers_test.go index 65fefe18b27f..dd15f07b0bc8 100644 --- a/test/e2e/v2/tests/hosted_cluster_image_mirroring_helpers_test.go +++ b/test/e2e/v2/tests/hosted_cluster_image_mirroring_helpers_test.go @@ -10,6 +10,7 @@ import ( "testing" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/test/e2e/v2/util/mirrorregistry" configv1 "github.com/openshift/api/config/v1" routev1 "github.com/openshift/api/route/v1" @@ -150,6 +151,70 @@ func TestImageMirroringConfigured(t *testing.T) { } } +func TestImageMirroringLoadSetup(t *testing.T) { + sentinel := errors.New("API unavailable") + fixtureRoute := imageMirroringTestRoute.DeepCopy() + fixtureRoute.Namespace, fixtureRoute.Name = mirrorregistry.DefaultNamespace, mirrorregistry.DefaultName + otherRoute := fixtureRoute.DeepCopy() + otherRoute.Namespace = "other" + invalid := imageMirroringTestHC.DeepCopy() + invalid.Spec.ImageContentSources = invalid.Spec.ImageContentSources[:1] + for _, tt := range []struct { + name string + hc *hyperv1.HostedCluster + objects []crclient.Object + failure error + wantConfigured bool + wantErr string + }{ + {name: "When the HC has imageContentSources and the fixture Route exists, it should return the validated setup", hc: imageMirroringTestHC, objects: []crclient.Object{fixtureRoute}, wantConfigured: true}, + {name: "When the fixture Route does not exist, it should report not configured without error so the spec skips", hc: imageMirroringTestHC}, + {name: "When only a Route in another namespace exists, it should report not configured without error", hc: imageMirroringTestHC, objects: []crclient.Object{otherRoute}}, + {name: "When the HC has no imageContentSources, it should report not configured without error so the spec skips", hc: &hyperv1.HostedCluster{}, objects: []crclient.Object{fixtureRoute}}, + {name: "When the Route read fails, it should return the error so the spec fails instead of skipping", hc: imageMirroringTestHC, failure: sentinel, wantErr: "API unavailable"}, + {name: "When the setup is present but invalid, it should return the error so the spec fails", hc: invalid, objects: []crclient.Object{fixtureRoute}, wantErr: "mirror-source"}, + } { + t.Run(tt.name, func(t *testing.T) { + scheme := runtime.NewScheme() + if err := routev1.Install(scheme); err != nil { + t.Fatal(err) + } + ctx := context.WithValue(t.Context(), imageMirroringContextKey{}, "spec context") + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tt.objects...).WithInterceptorFuncs(interceptor.Funcs{Get: func(got context.Context, c crclient.WithWatch, key crclient.ObjectKey, o crclient.Object, opts ...crclient.GetOption) error { + if got != ctx || key != (crclient.ObjectKey{Namespace: mirrorregistry.DefaultNamespace, Name: mirrorregistry.DefaultName}) { + t.Fatalf("wrong context or key: %v", key) + } + if tt.failure != nil { + return tt.failure + } + return c.Get(got, key, o, opts...) + }}).Build() + configured, expected, routeHost, err := imageMirroringLoadSetup(ctx, client, tt.hc) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) || configured { + t.Fatalf("expected error containing %q, got configured=%v err=%v", tt.wantErr, configured, err) + } + if tt.failure != nil && !errors.Is(err, tt.failure) { + t.Fatalf("lost API error: %v", err) + } + return + } + if err != nil || configured != tt.wantConfigured { + t.Fatalf("got configured=%v err=%v, want configured=%v", configured, err, tt.wantConfigured) + } + if !tt.wantConfigured { + if expected != nil || routeHost != "" { + t.Fatalf("unconfigured setup returned data: %+v %q", expected, routeHost) + } + return + } + if !reflect.DeepEqual(expected, tt.hc.Spec.ImageContentSources) || routeHost != fixtureRoute.Spec.Host { + t.Fatalf("got %+v %q", expected, routeHost) + } + }) + } +} + func TestImageMirroringCheckMappings(t *testing.T) { tests := []struct { name string diff --git a/test/e2e/v2/tests/hosted_cluster_image_mirroring_test.go b/test/e2e/v2/tests/hosted_cluster_image_mirroring_test.go index 106f749e7b75..b73035320087 100644 --- a/test/e2e/v2/tests/hosted_cluster_image_mirroring_test.go +++ b/test/e2e/v2/tests/hosted_cluster_image_mirroring_test.go @@ -14,40 +14,40 @@ import ( "github.com/openshift/hypershift/test/e2e/v2/internal" "github.com/openshift/hypershift/test/e2e/v2/util/mirrorregistry" - routev1 "github.com/openshift/api/route/v1" - + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" crclient "sigs.k8s.io/controller-runtime/pkg/client" ) // imageMirroringSetupOrSkip skips when the HC has no imageContentSources or the // mirror fixture Route does not exist. Once both are present, the setup must be -// valid: malformed, duplicate, or non-fixture mappings fail the spec. -func imageMirroringSetupOrSkip(tc *internal.TestContext) (*hyperv1.HostedCluster, []hyperv1.ImageContentSource) { +// valid: malformed, duplicate, or non-fixture mappings fail the spec, as do API +// errors. It also returns the fixture Route host. The decision logic lives in +// imageMirroringLoadSetup; this wrapper only asserts and skips. +func imageMirroringSetupOrSkip(g Gomega, tc *internal.TestContext) (*hyperv1.HostedCluster, []hyperv1.ImageContentSource, string) { GinkgoHelper() - Expect(tc).NotTo(BeNil(), "test context must be initialized") + g.Expect(tc).NotTo(BeNil(), "test context must be initialized") hc, err := tc.GetHostedCluster() - Expect(err).NotTo(HaveOccurred(), "read HC %s/%s", tc.ClusterNamespace, tc.ClusterName) - route := &routev1.Route{} - err = tc.MgmtClient.Get(tc.Context, crclient.ObjectKey{Namespace: mirrorregistry.DefaultNamespace, Name: mirrorregistry.DefaultName}, route) - if err != nil && !apierrors.IsNotFound(err) { - Expect(err).NotTo(HaveOccurred(), "read mirror fixture Route %s/%s", mirrorregistry.DefaultNamespace, mirrorregistry.DefaultName) - } - if !imageMirroringConfigured(hc, err == nil) { + g.Expect(err).NotTo(HaveOccurred(), "read HC %s/%s", tc.ClusterNamespace, tc.ClusterName) + g.Expect(hc).NotTo(BeNil(), "HC %s/%s", tc.ClusterNamespace, tc.ClusterName) + configured, expected, routeHost, err := imageMirroringLoadSetup(tc.Context, tc.MgmtClient, hc) + g.Expect(err).NotTo(HaveOccurred(), "HC %s/%s image-mirroring setup", hc.Namespace, hc.Name) + if !configured { Skip("image mirroring is not configured on this hosted cluster") } - expected, err := imageMirroringExpected(hc, route) - Expect(err).NotTo(HaveOccurred(), "HC %s/%s image-mirroring setup", hc.Namespace, hc.Name) - return hc, expected + return hc, expected, routeHost } -// RegisterHostedClusterImageMirroringTests registers independent, read-only -// lifecycle checks for the HCCO IDMS and CPO ignition propagation paths. +// RegisterHostedClusterImageMirroringTests registers independent lifecycle +// checks for the HCCO IDMS and CPO ignition propagation paths and for workload +// pulls through the mirror on data plane nodes. They never mutate the HC; the +// workload check only creates and deletes its own namespace and pods. func RegisterHostedClusterImageMirroringTests(getTestCtx internal.TestContextGetter) { It("should propagate imageContentSources to the hosted cluster IDMS with the worker role", func() { tc := getTestCtx() - hc, expected := imageMirroringSetupOrSkip(tc) + hc, expected, _ := imageMirroringSetupOrSkip(Default, tc) hostedClient, err := tc.GetHostedClusterClientViaPortForward(hc) Expect(err).NotTo(HaveOccurred(), "open hosted client for HC %s/%s", hc.Namespace, hc.Name) @@ -66,7 +66,7 @@ func RegisterHostedClusterImageMirroringTests(getTestCtx internal.TestContextGet It("should propagate imageContentSources independently to the control plane ignition ConfigMap", func() { tc := getTestCtx() - hc, expected := imageMirroringSetupOrSkip(tc) + hc, expected, _ := imageMirroringSetupOrSkip(Default, tc) Expect(tc.ControlPlaneNamespace).NotTo(BeEmpty(), "HC %s/%s must have a control plane namespace", hc.Namespace, hc.Name) Eventually(func(g Gomega, ctx context.Context) { @@ -80,6 +80,72 @@ func RegisterHostedClusterImageMirroringTests(getTestCtx internal.TestContextGet }).WithContext(tc.Context).WithTimeout(10*time.Minute).WithPolling(3*time.Second).Should(Succeed(), "HC %s/%s ConfigMap %s/%s decoded IDMS mappings and labels", hc.Namespace, hc.Name, tc.ControlPlaneNamespace, imageMirroringIgnitionName) }) + It("should pull a workload image by digest through the mirror when the source is unreachable", func() { + tc := getTestCtx() + hc, expected, routeHost := imageMirroringSetupOrSkip(Default, tc) + + // Expectations come from the HC spec and the pinned input image, never + // from the fixture result. + mirrorHost, mirrorRepository, err := imageMirroringVirtualMirror(expected, routeHost) + Expect(err).NotTo(HaveOccurred(), "HC %s/%s virtual source mirror", hc.Namespace, hc.Name) + digestRef, listDigest, err := imageMirroringDigestRef(imageMirroringVirtualSource, mirrorregistry.DefaultRegistryImage) + Expect(err).NotTo(HaveOccurred(), "build by-digest reference") + + // CRI-O reports the per-platform manifest digest for a manifest-list + // pull, so read the list from the mirror and verify it by content. + caPEM, err := imageMirroringTrustBundle(tc.Context, tc.MgmtClient, hc) + Expect(err).NotTo(HaveOccurred(), "HC %s/%s trust bundle", hc.Namespace, hc.Name) + httpClient, err := imageMirroringHTTPClient(caPEM) + Expect(err).NotTo(HaveOccurred(), "HC %s/%s trust bundle client", hc.Namespace, hc.Name) + manifestList, err := imageMirroringFetchManifest(tc.Context, httpClient, mirrorHost, mirrorRepository, listDigest) + Expect(err).NotTo(HaveOccurred(), "read manifest list %s from mirror %s/%s", listDigest, mirrorHost, mirrorRepository) + + hostedClient, err := tc.GetHostedClusterClientViaPortForward(hc) + Expect(err).NotTo(HaveOccurred(), "open hosted client for HC %s/%s", hc.Namespace, hc.Name) + DeferCleanup(hostedClient.Close) + + nodes := &corev1.NodeList{} + Expect(hostedClient.List(tc.Context, nodes)).To(Succeed(), "list nodes of HC %s/%s", hc.Namespace, hc.Name) + Expect(imageMirroringReadyLinuxNodes(nodes.Items)).To(BeNumerically(">", 0), "HC %s/%s must have a Ready linux node", hc.Namespace, hc.Name) + + namespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "e2e-image-mirroring-"}} + Expect(hostedClient.Create(tc.Context, namespace)).To(Succeed(), "create workload namespace in HC %s/%s", hc.Namespace, hc.Name) + DeferCleanup(func(ctx context.Context) { + if err := hostedClient.Delete(ctx, namespace); err != nil && !apierrors.IsNotFound(err) { + Expect(err).NotTo(HaveOccurred(), "cleanup: delete namespace %s", namespace.Name) + } + }) + + byDigest := imageMirroringWorkloadPod(namespace.Name, "by-digest", digestRef) + Expect(hostedClient.Create(tc.Context, byDigest)).To(Succeed(), "create pod %s/%s", byDigest.Namespace, byDigest.Name) + DeferCleanup(func(ctx context.Context) { + if err := hostedClient.Delete(ctx, byDigest); err != nil && !apierrors.IsNotFound(err) { + Expect(err).NotTo(HaveOccurred(), "cleanup: delete pod %s/%s", byDigest.Namespace, byDigest.Name) + } + }) + + // GET errors and unfinished states keep polling; a check error (a failure + // that retries cannot fix) stops at once. + waitObjective := fmt.Sprintf("pod %s/%s (%s) to pull through the mirror and succeed", byDigest.Namespace, byDigest.Name, digestRef) + Eventually(func(g Gomega, ctx context.Context) { + current := &corev1.Pod{} + g.Expect(hostedClient.Get(ctx, crclient.ObjectKeyFromObject(byDigest), current)).To(Succeed(), "get pod %s/%s", byDigest.Namespace, byDigest.Name) + done, reason, err := imageMirroringCheckPulledPod(current) + if err != nil { + StopTrying(waitObjective).Wrap(err).Now() + } + g.Expect(done).To(BeTrue(), reason) + }).WithContext(tc.Context).WithTimeout(10*time.Minute).WithPolling(5*time.Second).Should(Succeed(), waitObjective) + + pulled := &corev1.Pod{} + Expect(hostedClient.Get(tc.Context, crclient.ObjectKeyFromObject(byDigest), pulled)).To(Succeed(), "read pod %s/%s", byDigest.Namespace, byDigest.Name) + Expect(pulled.Spec.NodeName).NotTo(BeEmpty(), "pod %s/%s must be bound to a node", pulled.Namespace, pulled.Name) + node := &corev1.Node{} + Expect(hostedClient.Get(tc.Context, crclient.ObjectKey{Name: pulled.Spec.NodeName}, node)).To(Succeed(), "read node %s", pulled.Spec.NodeName) + platformDigest, err := imageMirroringPlatformDigest(manifestList, listDigest, node.Status.NodeInfo.OperatingSystem, node.Status.NodeInfo.Architecture) + Expect(err).NotTo(HaveOccurred(), "select the manifest for node %s", node.Name) + Expect(imageMirroringCheckImageID(pulled, []string{platformDigest, listDigest})).To(Succeed()) + }) } var _ = Describe("[sig-hypershift][Jira:Hypershift][Feature:ImageMirroring] Hosted Cluster Image Mirroring", Label("lifecycle", "image-mirroring", internal.InformingLabel), func() { diff --git a/test/e2e/v2/tests/hosted_cluster_image_mirroring_workload_helpers.go b/test/e2e/v2/tests/hosted_cluster_image_mirroring_workload_helpers.go new file mode 100644 index 000000000000..20154eb9048f --- /dev/null +++ b/test/e2e/v2/tests/hosted_cluster_image_mirroring_workload_helpers.go @@ -0,0 +1,336 @@ +//go:build e2ev2 + +package tests + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "regexp" + "strings" + "time" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + crclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + imageMirroringWorkloadContainer = "registry" + imageMirroringTrustBundleKey = "ca-bundle.crt" + imageMirroringManifestMaxBytes = 4 << 20 + imageMirroringManifestListType = "application/vnd.docker.distribution.manifest.list.v2+json" + imageMirroringOCIIndexType = "application/vnd.oci.image.index.v1+json" + imageMirroringHTTPClientTimeout = 30 * time.Second +) + +var imageMirroringDigestPattern = regexp.MustCompile(`^sha256:[a-f0-9]{64}$`) + +// imageMirroringVirtualMirror returns the host and repository path of the +// virtual source's mirror at the exact fixture Route host, as declared in the +// validated HC imageContentSources. +func imageMirroringVirtualMirror(expected []hyperv1.ImageContentSource, routeHost string) (string, string, error) { + sourceHost, _, _ := strings.Cut(imageMirroringVirtualSource, "/") + if !strings.HasSuffix(sourceHost, ".invalid") { + return "", "", fmt.Errorf("virtual source %q must use an unresolvable .invalid host", imageMirroringVirtualSource) + } + if strings.TrimSpace(routeHost) == "" { + return "", "", fmt.Errorf("mirror fixture Route host must not be empty") + } + for _, entry := range expected { + if entry.Source != imageMirroringVirtualSource { + continue + } + for _, mirror := range entry.Mirrors { + host, path, ok := strings.Cut(mirror, "/") + if ok && path != "" && host == routeHost { + return host, path, nil + } + } + return "", "", fmt.Errorf("HC source %q has no mirror at Route host %q: %v", entry.Source, routeHost, entry.Mirrors) + } + return "", "", fmt.Errorf("HC imageContentSources have no entry for virtual source %q: %+v", imageMirroringVirtualSource, expected) +} + +// imageMirroringDigestRef returns repository@digest, taking the digest from a +// by-digest image reference. +func imageMirroringDigestRef(repository, image string) (string, string, error) { + if strings.TrimSpace(repository) == "" || strings.Contains(repository, "@") { + return "", "", fmt.Errorf("repository %q must be nonblank and must not contain a digest", repository) + } + if lastSegment := repository[strings.LastIndex(repository, "/")+1:]; strings.Contains(lastSegment, ":") { + return "", "", fmt.Errorf("repository %q must not contain a tag", repository) + } + _, digest, ok := strings.Cut(image, "@") + if !ok || !imageMirroringDigestPattern.MatchString(digest) { + return "", "", fmt.Errorf("image %q must be referenced by a sha256 digest", image) + } + return repository + "@" + digest, digest, nil +} + +// imageMirroringImageIDDigest extracts the manifest digest from a container +// status imageID ("repo@sha256:..." or "docker-pullable://repo@sha256:..."). +// A bare image ID is rejected because it is not a manifest digest. +func imageMirroringImageIDDigest(imageID string) (string, error) { + ref := strings.TrimPrefix(imageID, "docker-pullable://") + at := strings.LastIndex(ref, "@") + if at <= 0 || !imageMirroringDigestPattern.MatchString(ref[at+1:]) { + return "", fmt.Errorf("imageID %q is not a repository@sha256 manifest reference", imageID) + } + return ref[at+1:], nil +} + +// imageMirroringTrustBundle reads the CA bundle referenced by the HC +// additionalTrustBundle from the HC namespace. +func imageMirroringTrustBundle(ctx context.Context, client crclient.Client, hc *hyperv1.HostedCluster) ([]byte, error) { + if hc == nil || hc.Spec.AdditionalTrustBundle == nil || strings.TrimSpace(hc.Spec.AdditionalTrustBundle.Name) == "" { + return nil, fmt.Errorf("HC must reference an additionalTrustBundle ConfigMap") + } + key := crclient.ObjectKey{Namespace: hc.Namespace, Name: hc.Spec.AdditionalTrustBundle.Name} + cm := &corev1.ConfigMap{} + if err := client.Get(ctx, key, cm); err != nil { + return nil, fmt.Errorf("read additionalTrustBundle ConfigMap %s: %w", key, err) + } + bundle := cm.Data[imageMirroringTrustBundleKey] + if strings.TrimSpace(bundle) == "" { + return nil, fmt.Errorf("additionalTrustBundle ConfigMap %s has an empty %s", key, imageMirroringTrustBundleKey) + } + return []byte(bundle), nil +} + +// imageMirroringHTTPClient returns an HTTPS client that trusts only caPEM. +func imageMirroringHTTPClient(caPEM []byte) (*http.Client, error) { + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + return nil, fmt.Errorf("trust bundle contains no PEM certificates") + } + return &http.Client{ + Timeout: imageMirroringHTTPClientTimeout, + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, + }, + }, nil +} + +// imageMirroringFetchManifest reads a manifest list or index by digest from the +// mirror's registry API. The body is returned unverified. +func imageMirroringFetchManifest(ctx context.Context, client *http.Client, host, repository, digest string) ([]byte, error) { + url := fmt.Sprintf("https://%s/v2/%s/manifests/%s", host, repository, digest) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("build request for %s: %w", url, err) + } + req.Header.Set("Accept", imageMirroringManifestListType+", "+imageMirroringOCIIndexType) + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("GET %s: %w", url, err) + } + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, imageMirroringManifestMaxBytes+1)) + if err != nil { + return nil, fmt.Errorf("read %s: %w", url, err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("GET %s: status %d: %.512s", url, resp.StatusCode, body) + } + if len(body) > imageMirroringManifestMaxBytes { + return nil, fmt.Errorf("GET %s: manifest exceeds %d bytes", url, imageMirroringManifestMaxBytes) + } + return body, nil +} + +// imageMirroringPlatformDigest verifies that body is the manifest list or +// index identified by listDigest and returns the digest of its single entry +// for os/architecture. +func imageMirroringPlatformDigest(body []byte, listDigest, os, architecture string) (string, error) { + sum := sha256.Sum256(body) + if got := "sha256:" + hex.EncodeToString(sum[:]); got != listDigest { + return "", fmt.Errorf("mirror manifest digest %s does not match requested digest %s", got, listDigest) + } + var list struct { + MediaType string `json:"mediaType"` + Manifests []struct { + Digest string `json:"digest"` + Platform *struct { + OS string `json:"os"` + Architecture string `json:"architecture"` + } `json:"platform"` + } `json:"manifests"` + } + if err := json.Unmarshal(body, &list); err != nil { + return "", fmt.Errorf("decode manifest list %s: %w", listDigest, err) + } + if (list.MediaType != imageMirroringManifestListType && list.MediaType != imageMirroringOCIIndexType && list.MediaType != "") || len(list.Manifests) == 0 { + return "", fmt.Errorf("manifest %s is not a nonempty manifest list or index (mediaType %q)", listDigest, list.MediaType) + } + var found []string + for _, m := range list.Manifests { + if m.Platform != nil && m.Platform.OS == os && m.Platform.Architecture == architecture { + found = append(found, m.Digest) + } + } + if len(found) != 1 { + return "", fmt.Errorf("manifest list %s has %d entries for %s/%s, expected exactly one", listDigest, len(found), os, architecture) + } + if !imageMirroringDigestPattern.MatchString(found[0]) { + return "", fmt.Errorf("manifest list %s has a malformed digest %q for %s/%s", listDigest, found[0], os, architecture) + } + return found[0], nil +} + +// imageMirroringReadyLinuxNodes counts Ready nodes labeled kubernetes.io/os=linux. +func imageMirroringReadyLinuxNodes(nodes []corev1.Node) int { + count := 0 + for _, node := range nodes { + if node.Labels[corev1.LabelOSStable] != "linux" { + continue + } + for _, cond := range node.Status.Conditions { + if cond.Type == corev1.NodeReady && cond.Status == corev1.ConditionTrue { + count++ + break + } + } + } + return count +} + +// imageMirroringWorkloadPod returns a restricted-compliant one-shot pod that +// always pulls image and runs the registry binary's version command. +func imageMirroringWorkloadPod(namespace, name, image string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + NodeSelector: map[string]string{corev1.LabelOSStable: "linux"}, + AutomountServiceAccountToken: ptr.To(false), + TerminationGracePeriodSeconds: ptr.To[int64](5), + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: ptr.To(true), + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + }, + Containers: []corev1.Container{{ + Name: imageMirroringWorkloadContainer, + Image: image, + ImagePullPolicy: corev1.PullAlways, + Command: []string{"/bin/registry", "--version"}, + Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("16Mi"), + }}, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + }, + }}, + }, + } +} + +// imageMirroringContainerStatus returns the workload container status, or nil +// when the pod is nil or has no status for that container. +func imageMirroringContainerStatus(pod *corev1.Pod) *corev1.ContainerStatus { + if pod == nil { + return nil + } + for i := range pod.Status.ContainerStatuses { + if pod.Status.ContainerStatuses[i].Name == imageMirroringWorkloadContainer { + return &pod.Status.ContainerStatuses[i] + } + } + return nil +} + +// imageMirroringPodSummary renders the pod state used in predicate reasons. +func imageMirroringPodSummary(pod *corev1.Pod) string { + if pod == nil { + return "pod " + } + summary := fmt.Sprintf("pod %s/%s phase=%q", pod.Namespace, pod.Name, pod.Status.Phase) + for _, cond := range pod.Status.Conditions { + if cond.Type == corev1.PodScheduled && cond.Status != corev1.ConditionTrue { + summary += fmt.Sprintf(" unscheduled=%s: %s", cond.Reason, cond.Message) + } + } + status := imageMirroringContainerStatus(pod) + switch { + case status == nil: + summary += fmt.Sprintf(" container %q has no status", imageMirroringWorkloadContainer) + case status.State.Waiting != nil: + summary += fmt.Sprintf(" waiting=%s: %s", status.State.Waiting.Reason, status.State.Waiting.Message) + case status.State.Terminated != nil: + summary += fmt.Sprintf(" terminated=%s exitCode=%d: %s", status.State.Terminated.Reason, status.State.Terminated.ExitCode, status.State.Terminated.Message) + case status.State.Running != nil: + summary += " running" + } + return summary +} + +// imageMirroringCheckPulledPod is done when the one-shot container exited 0 +// and the pod Succeeded. Failures that retries cannot fix are terminal errors. +func imageMirroringCheckPulledPod(pod *corev1.Pod) (bool, string, error) { + if pod == nil { + return false, "", fmt.Errorf("pod must not be nil") + } + summary := imageMirroringPodSummary(pod) + if pod.Status.Phase == corev1.PodFailed { + return false, "", fmt.Errorf("%s", summary) + } + status := imageMirroringContainerStatus(pod) + if status == nil { + return false, summary, nil + } + if waiting := status.State.Waiting; waiting != nil { + switch waiting.Reason { + case "CreateContainerConfigError", "CreateContainerError", "InvalidImageName", "ErrImageNeverPull": + return false, "", fmt.Errorf("%s", summary) + } + return false, summary, nil + } + if terminated := status.State.Terminated; terminated != nil { + if terminated.ExitCode != 0 { + return false, "", fmt.Errorf("%s", summary) + } + if pod.Status.Phase == corev1.PodSucceeded { + return true, summary, nil + } + } + return false, summary, nil +} + +// imageMirroringCheckImageID verifies that the container imageID digest is one +// of the accepted manifest digests. +func imageMirroringCheckImageID(pod *corev1.Pod, accepted []string) error { + if pod == nil { + return fmt.Errorf("pod must not be nil") + } + if len(accepted) == 0 { + return fmt.Errorf("no accepted digests for pod %s/%s", pod.Namespace, pod.Name) + } + status := imageMirroringContainerStatus(pod) + if status == nil { + return fmt.Errorf("pod %s/%s has no status for container %q", pod.Namespace, pod.Name, imageMirroringWorkloadContainer) + } + digest, err := imageMirroringImageIDDigest(status.ImageID) + if err != nil { + return fmt.Errorf("pod %s/%s: %w", pod.Namespace, pod.Name, err) + } + for _, want := range accepted { + if digest == want { + return nil + } + } + return fmt.Errorf("pod %s/%s imageID %q digest is not one of the mirror digests %v", pod.Namespace, pod.Name, status.ImageID, accepted) +} diff --git a/test/e2e/v2/tests/hosted_cluster_image_mirroring_workload_helpers_test.go b/test/e2e/v2/tests/hosted_cluster_image_mirroring_workload_helpers_test.go new file mode 100644 index 000000000000..c776729c68a9 --- /dev/null +++ b/test/e2e/v2/tests/hosted_cluster_image_mirroring_workload_helpers_test.go @@ -0,0 +1,503 @@ +//go:build e2ev2 + +package tests + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "encoding/hex" + "encoding/pem" + "math/big" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/test/e2e/v2/util/mirrorregistry" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +// imageMirroringTestManifestList is the verbatim manifest list of +// quay.io/libpod/registry:2.8.2 (skopeo inspect --raw). Its sha256 is the +// digest pinned in mirrorregistry.DefaultRegistryImage. +const imageMirroringTestManifestList = `{"manifests":[{"digest":"sha256:da1fbcd13a7ddc77d0d964a5c5c4cb707b5d440a028b0b42fe574b9e99077e27","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"architecture":"amd64","os":"linux"},"size":1363},{"digest":"sha256:ea32c4ea1ffa10d85c25b55e3e775a72013e685bf80f8da79ec5c5ddc4a4a63f","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"architecture":"arm","os":"linux","variant":"v6"},"size":1363},{"digest":"sha256:d72af0ec30bf3a5531f2d4dc4d0893f11b40cfd0a5dcbcbdcc19adde233847d2","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"architecture":"arm","os":"linux","variant":"v7"},"size":1363},{"digest":"sha256:fab2c6a4bb4acf3ee3ae611943e2596a73e9adb4da74432b1bcd686539f7f56f","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"architecture":"arm64","os":"linux","variant":"v8"},"size":1363},{"digest":"sha256:a353b4af55af90d0d30511d2191edb9da38f174390ce91e2329fa1839b6c2408","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"architecture":"ppc64le","os":"linux"},"size":1363},{"digest":"sha256:4cab339812e3bfa9e2242292f46ca2a99e804a4604d29f342f2b935381ebad3c","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"architecture":"s390x","os":"linux"},"size":1363}],"mediaType":"application\/vnd.docker.distribution.manifest.list.v2+json","schemaVersion":2}` + +const ( + imageMirroringTestListDigest = "sha256:e642e8604d305a3b82c8c1807b5df7a1a84cc650d57a60f9c5c2b78efec54b3f" + imageMirroringTestAMD64Digest = "sha256:da1fbcd13a7ddc77d0d964a5c5c4cb707b5d440a028b0b42fe574b9e99077e27" + imageMirroringTestARM64Digest = "sha256:fab2c6a4bb4acf3ee3ae611943e2596a73e9adb4da74432b1bcd686539f7f56f" +) + +func TestImageMirroringVirtualMirror(t *testing.T) { + tests := []struct { + name string + change func([]hyperv1.ImageContentSource) []hyperv1.ImageContentSource + routeHost string + wantHost string + wantPath string + wantErr string + }{ + {name: "When the virtual source has a mirror at the Route host, it should return its host and path", routeHost: "mirror.apps.ci.hypershift.dev", wantHost: "mirror.apps.ci.hypershift.dev", wantPath: "libpod/registry"}, + {name: "When the virtual source lists another mirror first, it should select the one at the Route host", routeHost: "mirror.apps.ci.hypershift.dev", wantHost: "mirror.apps.ci.hypershift.dev", wantPath: "libpod/registry", change: func(ics []hyperv1.ImageContentSource) []hyperv1.ImageContentSource { + ics[1].Mirrors = append([]string{"backup.apps.ci.hypershift.dev/libpod/registry"}, ics[1].Mirrors...) + return ics + }}, + {name: "When the Route host is empty, it should return an error", routeHost: " ", wantErr: "Route host"}, + {name: "When no mirror is at the Route host, it should return an error", routeHost: "other.apps.ci.hypershift.dev", wantErr: "no mirror at Route host"}, + {name: "When a mirror only shares the host prefix, it should return an error", routeHost: "mirror.apps.ci.hypershift.dev", wantErr: "no mirror at Route host", change: func(ics []hyperv1.ImageContentSource) []hyperv1.ImageContentSource { + ics[1].Mirrors = []string{"mirror.apps.ci.hypershift.dev.evil/libpod/registry"} + return ics + }}, + {name: "When the mirror has no repository path, it should return an error", routeHost: "mirror.apps.ci.hypershift.dev", wantErr: "no mirror at Route host", change: func(ics []hyperv1.ImageContentSource) []hyperv1.ImageContentSource { + ics[1].Mirrors = []string{"mirror.apps.ci.hypershift.dev"} + return ics + }}, + {name: "When the virtual source is missing, it should return an error", routeHost: "mirror.apps.ci.hypershift.dev", wantErr: "no entry for virtual source", change: func(ics []hyperv1.ImageContentSource) []hyperv1.ImageContentSource { + return ics[:1] + }}, + {name: "When the sources are nil, it should return an error", routeHost: "mirror.apps.ci.hypershift.dev", wantErr: "no entry for virtual source", change: func([]hyperv1.ImageContentSource) []hyperv1.ImageContentSource { + return nil + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ics := imageMirroringTestHC.DeepCopy().Spec.ImageContentSources + if tt.change != nil { + ics = tt.change(ics) + } + host, path, err := imageMirroringVirtualMirror(ics, tt.routeHost) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected error containing %q, got %v", tt.wantErr, err) + } + return + } + if err != nil || host != tt.wantHost || path != tt.wantPath { + t.Fatalf("got %q %q %v, want %q %q", host, path, err, tt.wantHost, tt.wantPath) + } + }) + } + t.Run("When the virtual source constant is inspected, it should use an unresolvable .invalid host", func(t *testing.T) { + host, _, _ := strings.Cut(imageMirroringVirtualSource, "/") + if !strings.HasSuffix(host, ".invalid") { + t.Fatalf("virtual source host %q must end in .invalid", host) + } + }) +} + +func TestImageMirroringDigestRef(t *testing.T) { + tests := []struct { + name string + repository string + image string + wantRef string + wantErr string + }{ + {name: "When the image is the pinned fixture image, it should build a by-digest reference on the repository", repository: imageMirroringVirtualSource, image: mirrorregistry.DefaultRegistryImage, wantRef: imageMirroringVirtualSource + "@" + imageMirroringTestListDigest}, + {name: "When the repository host has a port, it should accept it", repository: "registry.ci.local:5000/libpod/registry", image: mirrorregistry.DefaultRegistryImage, wantRef: "registry.ci.local:5000/libpod/registry@" + imageMirroringTestListDigest}, + {name: "When the image has no digest, it should return an error", repository: imageMirroringVirtualSource, image: "quay.io/libpod/registry:2.8.2", wantErr: "sha256 digest"}, + {name: "When the digest is malformed, it should return an error", repository: imageMirroringVirtualSource, image: "quay.io/libpod/registry@sha256:E642", wantErr: "sha256 digest"}, + {name: "When the repository is blank, it should return an error", repository: " ", image: mirrorregistry.DefaultRegistryImage, wantErr: "nonblank"}, + {name: "When the repository has a digest, it should return an error", repository: imageMirroringVirtualSource + "@" + imageMirroringTestListDigest, image: mirrorregistry.DefaultRegistryImage, wantErr: "digest"}, + {name: "When the repository has a tag, it should return an error", repository: imageMirroringVirtualSource + ":2.8.2", image: mirrorregistry.DefaultRegistryImage, wantErr: "tag"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ref, digest, err := imageMirroringDigestRef(tt.repository, tt.image) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected error containing %q, got %v", tt.wantErr, err) + } + return + } + if err != nil || ref != tt.wantRef || digest != imageMirroringTestListDigest { + t.Fatalf("got %q %q %v, want %q", ref, digest, err, tt.wantRef) + } + }) + } +} + +func TestImageMirroringImageIDDigest(t *testing.T) { + tests := []struct { + name string + imageID string + want string + wantErr bool + }{ + {name: "When the imageID is a CRI-O repository digest, it should return the digest", imageID: imageMirroringVirtualSource + "@" + imageMirroringTestAMD64Digest, want: imageMirroringTestAMD64Digest}, + {name: "When the imageID has the docker-pullable prefix, it should return the digest", imageID: "docker-pullable://" + imageMirroringVirtualSource + "@" + imageMirroringTestAMD64Digest, want: imageMirroringTestAMD64Digest}, + {name: "When the imageID is a bare image ID, it should return an error", imageID: imageMirroringTestAMD64Digest, wantErr: true}, + {name: "When the imageID is empty, it should return an error", imageID: "", wantErr: true}, + {name: "When the imageID digest is malformed, it should return an error", imageID: imageMirroringVirtualSource + "@sha256:abc", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := imageMirroringImageIDDigest(tt.imageID) + if tt.wantErr != (err != nil) || got != tt.want { + t.Fatalf("got %q %v, want %q (error %v)", got, err, tt.want, tt.wantErr) + } + }) + } +} + +func TestImageMirroringTrustBundle(t *testing.T) { + hc := &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: "clusters", Name: "image-mirroring"}, + Spec: hyperv1.HostedClusterSpec{AdditionalTrustBundle: &corev1.LocalObjectReference{Name: "user-ca-bundle"}}, + } + bundle := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: "clusters", Name: "user-ca-bundle"}, Data: map[string]string{"ca-bundle.crt": "-----BEGIN CERTIFICATE-----\n"}} + tests := []struct { + name string + hc *hyperv1.HostedCluster + objects []runtime.Object + want string + wantErr string + }{ + {name: "When the HC references a populated ConfigMap, it should return its CA bundle", hc: hc, objects: []runtime.Object{bundle}, want: "-----BEGIN CERTIFICATE-----\n"}, + {name: "When the HC is nil, it should return an error", hc: nil, wantErr: "additionalTrustBundle"}, + {name: "When the HC has no trust bundle, it should return an error", hc: &hyperv1.HostedCluster{}, wantErr: "additionalTrustBundle"}, + {name: "When the ConfigMap is missing, it should return an error", hc: hc, wantErr: "clusters/user-ca-bundle"}, + {name: "When the ConfigMap key is empty, it should return an error", hc: hc, objects: []runtime.Object{&corev1.ConfigMap{ObjectMeta: bundle.ObjectMeta}}, wantErr: "empty ca-bundle.crt"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := fake.NewClientBuilder().WithRuntimeObjects(tt.objects...).Build() + got, err := imageMirroringTrustBundle(t.Context(), client, tt.hc) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected error containing %q, got %v", tt.wantErr, err) + } + return + } + if err != nil || string(got) != tt.want { + t.Fatalf("got %q %v, want %q", got, err, tt.want) + } + }) + } +} + +func TestImageMirroringHTTPClient(t *testing.T) { + t.Run("When the trust bundle has no PEM certificate, it should return an error", func(t *testing.T) { + if _, err := imageMirroringHTTPClient([]byte("not a certificate")); err == nil { + t.Fatal("expected error") + } + }) + t.Run("When the trust bundle has a certificate, it should return a client with a timeout", func(t *testing.T) { + server := httptest.NewTLSServer(http.NotFoundHandler()) + defer server.Close() + client, err := imageMirroringHTTPClient(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})) + if err != nil || client.Timeout == 0 { + t.Fatalf("got %+v %v", client, err) + } + }) +} + +func TestImageMirroringFetchManifest(t *testing.T) { + const path = "/v2/mirror-source.hypershift-e2e.invalid/libpod/registry/manifests/" + imageMirroringTestListDigest + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != path { + http.Error(w, `{"errors":[{"code":"MANIFEST_UNKNOWN"}]}`, http.StatusNotFound) + return + } + accept := r.Header.Get("Accept") + if !strings.Contains(accept, imageMirroringManifestListType) || !strings.Contains(accept, imageMirroringOCIIndexType) { + http.Error(w, "missing list media types in Accept: "+accept, http.StatusNotAcceptable) + return + } + _, _ = w.Write([]byte(imageMirroringTestManifestList)) + }) + server := httptest.NewTLSServer(handler) + defer server.Close() + caOf := func(s *httptest.Server) []byte { + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: s.Certificate().Raw}) + } + host := strings.TrimPrefix(server.URL, "https://") + + tests := []struct { + name string + ca []byte + repository string + wantErr string + }{ + {name: "When the mirror serves the list with the trusted CA, it should return the exact body", ca: caOf(server), repository: imageMirroringVirtualSource}, + {name: "When the manifest is unknown, it should return the status and body", ca: caOf(server), repository: "libpod/other", wantErr: "status 404"}, + {name: "When the server certificate is not signed by the trust bundle, it should return a TLS error", ca: imageMirroringTestUnrelatedCA(t), repository: imageMirroringVirtualSource, wantErr: "certificate"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client, err := imageMirroringHTTPClient(tt.ca) + if err != nil { + t.Fatal(err) + } + body, err := imageMirroringFetchManifest(t.Context(), client, host, tt.repository, imageMirroringTestListDigest) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected error containing %q, got %v", tt.wantErr, err) + } + return + } + if err != nil || string(body) != imageMirroringTestManifestList { + t.Fatalf("got %d bytes, %v", len(body), err) + } + }) + } +} + +// imageMirroringTestUnrelatedCA returns a self-signed CA that did not sign +// the httptest certificate (all httptest servers share one certificate). +func imageMirroringTestUnrelatedCA(t *testing.T) []byte { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "unrelated-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) +} + +func TestImageMirroringPlatformDigest(t *testing.T) { + tests := []struct { + name string + body string + digest string + os string + arch string + want string + wantErr string + }{ + {name: "When the node is linux/amd64, it should return the amd64 manifest digest", body: imageMirroringTestManifestList, digest: imageMirroringTestListDigest, os: "linux", arch: "amd64", want: imageMirroringTestAMD64Digest}, + {name: "When the node is linux/arm64, it should return the arm64 manifest digest", body: imageMirroringTestManifestList, digest: imageMirroringTestListDigest, os: "linux", arch: "arm64", want: imageMirroringTestARM64Digest}, + {name: "When the body does not hash to the requested digest, it should return an error", body: imageMirroringTestManifestList + " ", digest: imageMirroringTestListDigest, os: "linux", arch: "amd64", wantErr: "does not match"}, + {name: "When several entries match the platform, it should return an error", body: imageMirroringTestManifestList, digest: imageMirroringTestListDigest, os: "linux", arch: "arm", wantErr: "2 entries"}, + {name: "When no entry matches the platform, it should return an error", body: imageMirroringTestManifestList, digest: imageMirroringTestListDigest, os: "windows", arch: "amd64", wantErr: "0 entries"}, + {name: "When the manifest is not a list, it should return an error", body: `{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json"}`, os: "linux", arch: "amd64", wantErr: "not a nonempty manifest list"}, + {name: "When the body is not JSON, it should return an error", body: "not json", os: "linux", arch: "amd64", wantErr: "decode"}, + {name: "When the matching entry digest is malformed, it should return an error", body: `{"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"digest":"sha256:abc","platform":{"os":"linux","architecture":"amd64"}}]}`, os: "linux", arch: "amd64", wantErr: "malformed"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + digest := tt.digest + if digest == "" { + digest = imageMirroringTestSHA256(tt.body) + } + got, err := imageMirroringPlatformDigest([]byte(tt.body), digest, tt.os, tt.arch) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected error containing %q, got %v", tt.wantErr, err) + } + return + } + if err != nil || got != tt.want { + t.Fatalf("got %q %v, want %q", got, err, tt.want) + } + }) + } + t.Run("When the test manifest list is hashed, it should match the pinned fixture image digest", func(t *testing.T) { + if _, digest, _ := strings.Cut(mirrorregistry.DefaultRegistryImage, "@"); imageMirroringTestSHA256(imageMirroringTestManifestList) != digest { + t.Fatalf("test manifest list does not hash to %s", digest) + } + }) +} + +func imageMirroringTestSHA256(body string) string { + sum := sha256.Sum256([]byte(body)) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func TestImageMirroringReadyLinuxNodes(t *testing.T) { + node := func(os string, ready corev1.ConditionStatus) corev1.Node { + return corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{corev1.LabelOSStable: os}}, + Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: ready}}}, + } + } + tests := []struct { + name string + nodes []corev1.Node + want int + }{ + {name: "When nodes are Ready linux nodes, it should count them", nodes: []corev1.Node{node("linux", corev1.ConditionTrue), node("linux", corev1.ConditionTrue)}, want: 2}, + {name: "When a linux node is not Ready, it should not count it", nodes: []corev1.Node{node("linux", corev1.ConditionFalse), node("linux", corev1.ConditionUnknown)}, want: 0}, + {name: "When a Ready node is not linux, it should not count it", nodes: []corev1.Node{node("windows", corev1.ConditionTrue), {}}, want: 0}, + {name: "When there are no nodes, it should return zero", want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := imageMirroringReadyLinuxNodes(tt.nodes); got != tt.want { + t.Fatalf("got %d, want %d", got, tt.want) + } + }) + } +} + +func TestImageMirroringWorkloadPod(t *testing.T) { + t.Run("When a workload pod is built, it should be a restricted one-shot pod that always pulls the image", func(t *testing.T) { + pod := imageMirroringWorkloadPod("e2e-image-mirroring-x", "by-digest", imageMirroringVirtualSource+"@"+imageMirroringTestListDigest) + if pod.Namespace != "e2e-image-mirroring-x" || pod.Name != "by-digest" || pod.Spec.RestartPolicy != corev1.RestartPolicyNever { + t.Fatalf("unexpected identity or restart policy: %+v", pod.ObjectMeta) + } + if pod.Spec.NodeSelector[corev1.LabelOSStable] != "linux" || pod.Spec.AutomountServiceAccountToken == nil || *pod.Spec.AutomountServiceAccountToken { + t.Fatalf("expected linux node selector and no service account token: %+v", pod.Spec) + } + psc := pod.Spec.SecurityContext + if psc == nil || psc.RunAsNonRoot == nil || !*psc.RunAsNonRoot || psc.RunAsUser != nil || psc.SeccompProfile == nil || psc.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault { + t.Fatalf("pod security context is not restricted: %+v", psc) + } + if len(pod.Spec.Containers) != 1 { + t.Fatalf("expected one container, got %d", len(pod.Spec.Containers)) + } + c := pod.Spec.Containers[0] + if c.Name != imageMirroringWorkloadContainer || c.Image != imageMirroringVirtualSource+"@"+imageMirroringTestListDigest || c.ImagePullPolicy != corev1.PullAlways { + t.Fatalf("unexpected container image settings: %+v", c) + } + if !reflect.DeepEqual(c.Command, []string{"/bin/registry", "--version"}) { + t.Fatalf("unexpected command %v", c.Command) + } + csc := c.SecurityContext + if csc == nil || csc.AllowPrivilegeEscalation == nil || *csc.AllowPrivilegeEscalation || csc.Capabilities == nil || !reflect.DeepEqual(csc.Capabilities.Drop, []corev1.Capability{"ALL"}) { + t.Fatalf("container security context is not restricted: %+v", csc) + } + }) +} + +func imageMirroringTestPod(phase corev1.PodPhase, state *corev1.ContainerState) *corev1.Pod { + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "e2e-image-mirroring-x", Name: "pod"}, Status: corev1.PodStatus{Phase: phase}} + if state != nil { + pod.Status.ContainerStatuses = []corev1.ContainerStatus{{Name: imageMirroringWorkloadContainer, State: *state}} + } + return pod +} + +func imageMirroringWaitingState(reason, message string) *corev1.ContainerState { + return &corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: reason, Message: message}} +} + +func imageMirroringTerminatedState(exitCode int32) *corev1.ContainerState { + return &corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: "Completed", ExitCode: exitCode}} +} + +func imageMirroringRunningState() *corev1.ContainerState { + return &corev1.ContainerState{Running: &corev1.ContainerStateRunning{}} +} + +func TestImageMirroringCheckPulledPod(t *testing.T) { + unschedulable := imageMirroringTestPod(corev1.PodPending, nil) + unschedulable.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: "Unschedulable", Message: "0/1 nodes are available"}} + otherContainer := imageMirroringTestPod(corev1.PodPending, nil) + otherContainer.Status.ContainerStatuses = []corev1.ContainerStatus{{Name: "other", State: *imageMirroringTerminatedState(0)}} + + tests := []struct { + name string + pod *corev1.Pod + wantDone bool + wantReason string + wantErr string + }{ + {name: "When the container exited 0 and the pod Succeeded, it should be done", pod: imageMirroringTestPod(corev1.PodSucceeded, imageMirroringTerminatedState(0)), wantDone: true}, + {name: "When the pod is Pending without container statuses, it should keep polling", pod: imageMirroringTestPod(corev1.PodPending, nil), wantReason: "has no status"}, + {name: "When the pod is unschedulable, it should keep polling and report the reason", pod: unschedulable, wantReason: "Unschedulable: 0/1 nodes are available"}, + {name: "When only another container has a status, it should keep polling", pod: otherContainer, wantReason: "has no status"}, + {name: "When the container is creating, it should keep polling", pod: imageMirroringTestPod(corev1.PodPending, imageMirroringWaitingState("ContainerCreating", "")), wantReason: "ContainerCreating"}, + {name: "When the image pull fails, it should keep polling and report the message", pod: imageMirroringTestPod(corev1.PodPending, imageMirroringWaitingState("ErrImagePull", "no such host")), wantReason: "ErrImagePull: no such host"}, + {name: "When the image pull backs off, it should keep polling", pod: imageMirroringTestPod(corev1.PodPending, imageMirroringWaitingState("ImagePullBackOff", "")), wantReason: "ImagePullBackOff"}, + {name: "When the container is running, it should keep polling", pod: imageMirroringTestPod(corev1.PodRunning, imageMirroringRunningState()), wantReason: "running"}, + {name: "When the container exited 0 but the pod has not Succeeded yet, it should keep polling", pod: imageMirroringTestPod(corev1.PodRunning, imageMirroringTerminatedState(0)), wantReason: "exitCode=0"}, + {name: "When the container config cannot be created, it should fail terminally", pod: imageMirroringTestPod(corev1.PodPending, imageMirroringWaitingState("CreateContainerConfigError", "container has runAsNonRoot and image will run as root")), wantErr: "CreateContainerConfigError"}, + {name: "When the image name is invalid, it should fail terminally", pod: imageMirroringTestPod(corev1.PodPending, imageMirroringWaitingState("InvalidImageName", "")), wantErr: "InvalidImageName"}, + {name: "When the container exits non-zero, it should fail terminally", pod: imageMirroringTestPod(corev1.PodRunning, imageMirroringTerminatedState(2)), wantErr: "exitCode=2"}, + {name: "When the pod Failed, it should fail terminally", pod: imageMirroringTestPod(corev1.PodFailed, nil), wantErr: "Failed"}, + {name: "When the pod is nil, it should fail terminally", pod: nil, wantErr: "nil"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + done, reason, err := imageMirroringCheckPulledPod(tt.pod) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) || done { + t.Fatalf("expected terminal error containing %q, got %v %v", tt.wantErr, done, err) + } + return + } + if err != nil || done != tt.wantDone || !strings.Contains(reason, tt.wantReason) || !strings.Contains(reason, "e2e-image-mirroring-x/pod") { + t.Fatalf("got %v %q %v, want %v containing %q", done, reason, err, tt.wantDone, tt.wantReason) + } + }) + } +} + +func TestImageMirroringPodState(t *testing.T) { + t.Run("When the pod is nil, it should not panic and report no container status", func(t *testing.T) { + if got := imageMirroringContainerStatus(nil); got != nil { + t.Fatalf("expected nil status, got %+v", got) + } + if got := imageMirroringPodSummary(nil); !strings.Contains(got, "") { + t.Fatalf("expected a nil pod summary, got %q", got) + } + }) + t.Run("When the workload container has a status, it should return that status", func(t *testing.T) { + pod := imageMirroringTestPod(corev1.PodRunning, imageMirroringRunningState()) + if got := imageMirroringContainerStatus(pod); got == nil || got.Name != imageMirroringWorkloadContainer { + t.Fatalf("expected the %q status, got %+v", imageMirroringWorkloadContainer, got) + } + }) +} + +func TestImageMirroringCheckImageID(t *testing.T) { + withImageID := func(imageID string) *corev1.Pod { + pod := imageMirroringTestPod(corev1.PodSucceeded, imageMirroringTerminatedState(0)) + pod.Status.ContainerStatuses[0].ImageID = imageID + return pod + } + accepted := []string{imageMirroringTestAMD64Digest, imageMirroringTestListDigest} + tests := []struct { + name string + pod *corev1.Pod + accepted []string + wantErr string + }{ + {name: "When the imageID has the per-platform digest, it should succeed", pod: withImageID(imageMirroringVirtualSource + "@" + imageMirroringTestAMD64Digest), accepted: accepted}, + {name: "When the imageID has the manifest list digest, it should succeed", pod: withImageID(imageMirroringVirtualSource + "@" + imageMirroringTestListDigest), accepted: accepted}, + {name: "When the imageID has another platform digest, it should return an error", pod: withImageID(imageMirroringVirtualSource + "@" + imageMirroringTestARM64Digest), accepted: accepted, wantErr: "not one of the mirror digests"}, + {name: "When the imageID is a bare image ID, it should return an error", pod: withImageID(imageMirroringTestAMD64Digest), accepted: accepted, wantErr: "manifest reference"}, + {name: "When the container has no status, it should return an error", pod: imageMirroringTestPod(corev1.PodSucceeded, nil), accepted: accepted, wantErr: "no status"}, + {name: "When no digests are accepted, it should return an error", pod: withImageID(imageMirroringVirtualSource + "@" + imageMirroringTestAMD64Digest), wantErr: "no accepted digests"}, + {name: "When the pod is nil, it should return an error", pod: nil, accepted: accepted, wantErr: "nil"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := imageMirroringCheckImageID(tt.pod, tt.accepted) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected error containing %q, got %v", tt.wantErr, err) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + } +}