From 122c4ec11c70dd3993b6d34203d798aefeafb621 Mon Sep 17 00:00:00 2001 From: Alessandro Rossi Date: Wed, 28 Oct 2026 10:45:42 +0100 Subject: [PATCH 1/2] test(e2e): verify workload pulls through the image mirror Prove that data plane nodes consume the IDMS registries.conf generated from HostedCluster imageContentSources, not only that the policy propagates. Run a restricted one-shot pod that references the fixture's virtual source (an unresolvable .invalid host) by digest with imagePullPolicy Always, so it can only start if the node pulled from the mirror. CRI-O reports the per-platform manifest digest for a manifest-list pull, so read the list from the mirror endpoint declared in the HC spec, trusting only the HC additionalTrustBundle, verify it hashes to the requested digest, and require the container imageID to match the node's platform entry. The existing setup gate now takes a Gomega instance and also returns the fixture Route host. Its decision logic moves to imageMirroringLoadSetup, which returns errors instead of asserting, so a missing Route still skips while an API error or an invalid setup fails. Cover it and the reference, manifest, trust bundle, and pod state helpers with unit tests. Signed-off-by: Alessandro Rossi Assisted-by: LLM --- .../hosted_cluster_image_mirroring_helpers.go | 32 +- ...ed_cluster_image_mirroring_helpers_test.go | 65 +++ .../hosted_cluster_image_mirroring_test.go | 104 +++- ...luster_image_mirroring_workload_helpers.go | 336 ++++++++++++ ...r_image_mirroring_workload_helpers_test.go | 503 ++++++++++++++++++ 5 files changed, 1020 insertions(+), 20 deletions(-) create mode 100644 test/e2e/v2/tests/hosted_cluster_image_mirroring_workload_helpers.go create mode 100644 test/e2e/v2/tests/hosted_cluster_image_mirroring_workload_helpers_test.go diff --git a/test/e2e/v2/tests/hosted_cluster_image_mirroring_helpers.go b/test/e2e/v2/tests/hosted_cluster_image_mirroring_helpers.go index 0d8a09886ea1..496f2168ebca 100644 --- a/test/e2e/v2/tests/hosted_cluster_image_mirroring_helpers.go +++ b/test/e2e/v2/tests/hosted_cluster_image_mirroring_helpers.go @@ -9,11 +9,13 @@ import ( "strings" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/test/e2e/v2/util/mirrorregistry" configv1 "github.com/openshift/api/config/v1" routev1 "github.com/openshift/api/route/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/sets" crclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -24,6 +26,11 @@ const ( imageMirroringIgnitionName = "ignition-config-40-image-content-source" imageMirroringWorkerLabel = "machineconfiguration.openshift.io/role" imageMirroringCoreLabel = "hypershift.openshift.io/core-ignition-config" + + // imageMirroringVirtualSource is the fixture source repository that only + // exists on the mirror: its ".invalid" host never resolves (RFC 2606), so a + // successful pull from it can only have been served by an IDMS mirror. + imageMirroringVirtualSource = "mirror-source.hypershift-e2e.invalid/libpod/registry" ) // imageMirroringConfigured reports whether the HC opted into image mirroring @@ -33,6 +40,29 @@ func imageMirroringConfigured(hc *hyperv1.HostedCluster, routeFound bool) bool { return hc != nil && len(hc.Spec.ImageContentSources) > 0 && routeFound } +// imageMirroringLoadSetup reads the mirror fixture Route from the management +// cluster and validates the HC image-mirroring setup against it. It returns +// configured=false and no error when the HC has no imageContentSources or the +// Route does not exist, so the caller can skip. A Route read error other than +// NotFound, or an invalid setup once both are present, is returned as an error +// so the caller fails instead of skipping. +func imageMirroringLoadSetup(ctx context.Context, client crclient.Client, hc *hyperv1.HostedCluster) (bool, []hyperv1.ImageContentSource, string, error) { + key := crclient.ObjectKey{Namespace: mirrorregistry.DefaultNamespace, Name: mirrorregistry.DefaultName} + route := &routev1.Route{} + err := client.Get(ctx, key, route) + if err != nil && !apierrors.IsNotFound(err) { + return false, nil, "", fmt.Errorf("read mirror fixture Route %s: %w", key, err) + } + if !imageMirroringConfigured(hc, err == nil) { + return false, nil, "", nil + } + expected, err := imageMirroringExpected(hc, route) + if err != nil { + return false, nil, "", err + } + return true, expected, route.Spec.Host, nil +} + // Expectations come only from a validated snapshot of HC input, never from a // producer conversion or either output. Mirror priority and multiplicity matter. func imageMirroringExpected(hc *hyperv1.HostedCluster, route *routev1.Route) ([]hyperv1.ImageContentSource, error) { @@ -58,7 +88,7 @@ func imageMirroringExpected(hc *hyperv1.HostedCluster, route *routev1.Route) ([] if route == nil || strings.TrimSpace(route.Spec.Host) == "" { return nil, fmt.Errorf("mirror fixture Route has an empty host") } - for _, source := range []string{"quay.io/libpod/registry", "mirror-source.hypershift-e2e.invalid/libpod/registry"} { + for _, source := range []string{"quay.io/libpod/registry", imageMirroringVirtualSource} { found := false for _, entry := range expected { if entry.Source != source { diff --git a/test/e2e/v2/tests/hosted_cluster_image_mirroring_helpers_test.go b/test/e2e/v2/tests/hosted_cluster_image_mirroring_helpers_test.go index 65fefe18b27f..dd15f07b0bc8 100644 --- a/test/e2e/v2/tests/hosted_cluster_image_mirroring_helpers_test.go +++ b/test/e2e/v2/tests/hosted_cluster_image_mirroring_helpers_test.go @@ -10,6 +10,7 @@ import ( "testing" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/test/e2e/v2/util/mirrorregistry" configv1 "github.com/openshift/api/config/v1" routev1 "github.com/openshift/api/route/v1" @@ -150,6 +151,70 @@ func TestImageMirroringConfigured(t *testing.T) { } } +func TestImageMirroringLoadSetup(t *testing.T) { + sentinel := errors.New("API unavailable") + fixtureRoute := imageMirroringTestRoute.DeepCopy() + fixtureRoute.Namespace, fixtureRoute.Name = mirrorregistry.DefaultNamespace, mirrorregistry.DefaultName + otherRoute := fixtureRoute.DeepCopy() + otherRoute.Namespace = "other" + invalid := imageMirroringTestHC.DeepCopy() + invalid.Spec.ImageContentSources = invalid.Spec.ImageContentSources[:1] + for _, tt := range []struct { + name string + hc *hyperv1.HostedCluster + objects []crclient.Object + failure error + wantConfigured bool + wantErr string + }{ + {name: "When the HC has imageContentSources and the fixture Route exists, it should return the validated setup", hc: imageMirroringTestHC, objects: []crclient.Object{fixtureRoute}, wantConfigured: true}, + {name: "When the fixture Route does not exist, it should report not configured without error so the spec skips", hc: imageMirroringTestHC}, + {name: "When only a Route in another namespace exists, it should report not configured without error", hc: imageMirroringTestHC, objects: []crclient.Object{otherRoute}}, + {name: "When the HC has no imageContentSources, it should report not configured without error so the spec skips", hc: &hyperv1.HostedCluster{}, objects: []crclient.Object{fixtureRoute}}, + {name: "When the Route read fails, it should return the error so the spec fails instead of skipping", hc: imageMirroringTestHC, failure: sentinel, wantErr: "API unavailable"}, + {name: "When the setup is present but invalid, it should return the error so the spec fails", hc: invalid, objects: []crclient.Object{fixtureRoute}, wantErr: "mirror-source"}, + } { + t.Run(tt.name, func(t *testing.T) { + scheme := runtime.NewScheme() + if err := routev1.Install(scheme); err != nil { + t.Fatal(err) + } + ctx := context.WithValue(t.Context(), imageMirroringContextKey{}, "spec context") + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tt.objects...).WithInterceptorFuncs(interceptor.Funcs{Get: func(got context.Context, c crclient.WithWatch, key crclient.ObjectKey, o crclient.Object, opts ...crclient.GetOption) error { + if got != ctx || key != (crclient.ObjectKey{Namespace: mirrorregistry.DefaultNamespace, Name: mirrorregistry.DefaultName}) { + t.Fatalf("wrong context or key: %v", key) + } + if tt.failure != nil { + return tt.failure + } + return c.Get(got, key, o, opts...) + }}).Build() + configured, expected, routeHost, err := imageMirroringLoadSetup(ctx, client, tt.hc) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) || configured { + t.Fatalf("expected error containing %q, got configured=%v err=%v", tt.wantErr, configured, err) + } + if tt.failure != nil && !errors.Is(err, tt.failure) { + t.Fatalf("lost API error: %v", err) + } + return + } + if err != nil || configured != tt.wantConfigured { + t.Fatalf("got configured=%v err=%v, want configured=%v", configured, err, tt.wantConfigured) + } + if !tt.wantConfigured { + if expected != nil || routeHost != "" { + t.Fatalf("unconfigured setup returned data: %+v %q", expected, routeHost) + } + return + } + if !reflect.DeepEqual(expected, tt.hc.Spec.ImageContentSources) || routeHost != fixtureRoute.Spec.Host { + t.Fatalf("got %+v %q", expected, routeHost) + } + }) + } +} + func TestImageMirroringCheckMappings(t *testing.T) { tests := []struct { name string diff --git a/test/e2e/v2/tests/hosted_cluster_image_mirroring_test.go b/test/e2e/v2/tests/hosted_cluster_image_mirroring_test.go index 106f749e7b75..b73035320087 100644 --- a/test/e2e/v2/tests/hosted_cluster_image_mirroring_test.go +++ b/test/e2e/v2/tests/hosted_cluster_image_mirroring_test.go @@ -14,40 +14,40 @@ import ( "github.com/openshift/hypershift/test/e2e/v2/internal" "github.com/openshift/hypershift/test/e2e/v2/util/mirrorregistry" - routev1 "github.com/openshift/api/route/v1" - + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" crclient "sigs.k8s.io/controller-runtime/pkg/client" ) // imageMirroringSetupOrSkip skips when the HC has no imageContentSources or the // mirror fixture Route does not exist. Once both are present, the setup must be -// valid: malformed, duplicate, or non-fixture mappings fail the spec. -func imageMirroringSetupOrSkip(tc *internal.TestContext) (*hyperv1.HostedCluster, []hyperv1.ImageContentSource) { +// valid: malformed, duplicate, or non-fixture mappings fail the spec, as do API +// errors. It also returns the fixture Route host. The decision logic lives in +// imageMirroringLoadSetup; this wrapper only asserts and skips. +func imageMirroringSetupOrSkip(g Gomega, tc *internal.TestContext) (*hyperv1.HostedCluster, []hyperv1.ImageContentSource, string) { GinkgoHelper() - Expect(tc).NotTo(BeNil(), "test context must be initialized") + g.Expect(tc).NotTo(BeNil(), "test context must be initialized") hc, err := tc.GetHostedCluster() - Expect(err).NotTo(HaveOccurred(), "read HC %s/%s", tc.ClusterNamespace, tc.ClusterName) - route := &routev1.Route{} - err = tc.MgmtClient.Get(tc.Context, crclient.ObjectKey{Namespace: mirrorregistry.DefaultNamespace, Name: mirrorregistry.DefaultName}, route) - if err != nil && !apierrors.IsNotFound(err) { - Expect(err).NotTo(HaveOccurred(), "read mirror fixture Route %s/%s", mirrorregistry.DefaultNamespace, mirrorregistry.DefaultName) - } - if !imageMirroringConfigured(hc, err == nil) { + g.Expect(err).NotTo(HaveOccurred(), "read HC %s/%s", tc.ClusterNamespace, tc.ClusterName) + g.Expect(hc).NotTo(BeNil(), "HC %s/%s", tc.ClusterNamespace, tc.ClusterName) + configured, expected, routeHost, err := imageMirroringLoadSetup(tc.Context, tc.MgmtClient, hc) + g.Expect(err).NotTo(HaveOccurred(), "HC %s/%s image-mirroring setup", hc.Namespace, hc.Name) + if !configured { Skip("image mirroring is not configured on this hosted cluster") } - expected, err := imageMirroringExpected(hc, route) - Expect(err).NotTo(HaveOccurred(), "HC %s/%s image-mirroring setup", hc.Namespace, hc.Name) - return hc, expected + return hc, expected, routeHost } -// RegisterHostedClusterImageMirroringTests registers independent, read-only -// lifecycle checks for the HCCO IDMS and CPO ignition propagation paths. +// RegisterHostedClusterImageMirroringTests registers independent lifecycle +// checks for the HCCO IDMS and CPO ignition propagation paths and for workload +// pulls through the mirror on data plane nodes. They never mutate the HC; the +// workload check only creates and deletes its own namespace and pods. func RegisterHostedClusterImageMirroringTests(getTestCtx internal.TestContextGetter) { It("should propagate imageContentSources to the hosted cluster IDMS with the worker role", func() { tc := getTestCtx() - hc, expected := imageMirroringSetupOrSkip(tc) + hc, expected, _ := imageMirroringSetupOrSkip(Default, tc) hostedClient, err := tc.GetHostedClusterClientViaPortForward(hc) Expect(err).NotTo(HaveOccurred(), "open hosted client for HC %s/%s", hc.Namespace, hc.Name) @@ -66,7 +66,7 @@ func RegisterHostedClusterImageMirroringTests(getTestCtx internal.TestContextGet It("should propagate imageContentSources independently to the control plane ignition ConfigMap", func() { tc := getTestCtx() - hc, expected := imageMirroringSetupOrSkip(tc) + hc, expected, _ := imageMirroringSetupOrSkip(Default, tc) Expect(tc.ControlPlaneNamespace).NotTo(BeEmpty(), "HC %s/%s must have a control plane namespace", hc.Namespace, hc.Name) Eventually(func(g Gomega, ctx context.Context) { @@ -80,6 +80,72 @@ func RegisterHostedClusterImageMirroringTests(getTestCtx internal.TestContextGet }).WithContext(tc.Context).WithTimeout(10*time.Minute).WithPolling(3*time.Second).Should(Succeed(), "HC %s/%s ConfigMap %s/%s decoded IDMS mappings and labels", hc.Namespace, hc.Name, tc.ControlPlaneNamespace, imageMirroringIgnitionName) }) + It("should pull a workload image by digest through the mirror when the source is unreachable", func() { + tc := getTestCtx() + hc, expected, routeHost := imageMirroringSetupOrSkip(Default, tc) + + // Expectations come from the HC spec and the pinned input image, never + // from the fixture result. + mirrorHost, mirrorRepository, err := imageMirroringVirtualMirror(expected, routeHost) + Expect(err).NotTo(HaveOccurred(), "HC %s/%s virtual source mirror", hc.Namespace, hc.Name) + digestRef, listDigest, err := imageMirroringDigestRef(imageMirroringVirtualSource, mirrorregistry.DefaultRegistryImage) + Expect(err).NotTo(HaveOccurred(), "build by-digest reference") + + // CRI-O reports the per-platform manifest digest for a manifest-list + // pull, so read the list from the mirror and verify it by content. + caPEM, err := imageMirroringTrustBundle(tc.Context, tc.MgmtClient, hc) + Expect(err).NotTo(HaveOccurred(), "HC %s/%s trust bundle", hc.Namespace, hc.Name) + httpClient, err := imageMirroringHTTPClient(caPEM) + Expect(err).NotTo(HaveOccurred(), "HC %s/%s trust bundle client", hc.Namespace, hc.Name) + manifestList, err := imageMirroringFetchManifest(tc.Context, httpClient, mirrorHost, mirrorRepository, listDigest) + Expect(err).NotTo(HaveOccurred(), "read manifest list %s from mirror %s/%s", listDigest, mirrorHost, mirrorRepository) + + hostedClient, err := tc.GetHostedClusterClientViaPortForward(hc) + Expect(err).NotTo(HaveOccurred(), "open hosted client for HC %s/%s", hc.Namespace, hc.Name) + DeferCleanup(hostedClient.Close) + + nodes := &corev1.NodeList{} + Expect(hostedClient.List(tc.Context, nodes)).To(Succeed(), "list nodes of HC %s/%s", hc.Namespace, hc.Name) + Expect(imageMirroringReadyLinuxNodes(nodes.Items)).To(BeNumerically(">", 0), "HC %s/%s must have a Ready linux node", hc.Namespace, hc.Name) + + namespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "e2e-image-mirroring-"}} + Expect(hostedClient.Create(tc.Context, namespace)).To(Succeed(), "create workload namespace in HC %s/%s", hc.Namespace, hc.Name) + DeferCleanup(func(ctx context.Context) { + if err := hostedClient.Delete(ctx, namespace); err != nil && !apierrors.IsNotFound(err) { + Expect(err).NotTo(HaveOccurred(), "cleanup: delete namespace %s", namespace.Name) + } + }) + + byDigest := imageMirroringWorkloadPod(namespace.Name, "by-digest", digestRef) + Expect(hostedClient.Create(tc.Context, byDigest)).To(Succeed(), "create pod %s/%s", byDigest.Namespace, byDigest.Name) + DeferCleanup(func(ctx context.Context) { + if err := hostedClient.Delete(ctx, byDigest); err != nil && !apierrors.IsNotFound(err) { + Expect(err).NotTo(HaveOccurred(), "cleanup: delete pod %s/%s", byDigest.Namespace, byDigest.Name) + } + }) + + // GET errors and unfinished states keep polling; a check error (a failure + // that retries cannot fix) stops at once. + waitObjective := fmt.Sprintf("pod %s/%s (%s) to pull through the mirror and succeed", byDigest.Namespace, byDigest.Name, digestRef) + Eventually(func(g Gomega, ctx context.Context) { + current := &corev1.Pod{} + g.Expect(hostedClient.Get(ctx, crclient.ObjectKeyFromObject(byDigest), current)).To(Succeed(), "get pod %s/%s", byDigest.Namespace, byDigest.Name) + done, reason, err := imageMirroringCheckPulledPod(current) + if err != nil { + StopTrying(waitObjective).Wrap(err).Now() + } + g.Expect(done).To(BeTrue(), reason) + }).WithContext(tc.Context).WithTimeout(10*time.Minute).WithPolling(5*time.Second).Should(Succeed(), waitObjective) + + pulled := &corev1.Pod{} + Expect(hostedClient.Get(tc.Context, crclient.ObjectKeyFromObject(byDigest), pulled)).To(Succeed(), "read pod %s/%s", byDigest.Namespace, byDigest.Name) + Expect(pulled.Spec.NodeName).NotTo(BeEmpty(), "pod %s/%s must be bound to a node", pulled.Namespace, pulled.Name) + node := &corev1.Node{} + Expect(hostedClient.Get(tc.Context, crclient.ObjectKey{Name: pulled.Spec.NodeName}, node)).To(Succeed(), "read node %s", pulled.Spec.NodeName) + platformDigest, err := imageMirroringPlatformDigest(manifestList, listDigest, node.Status.NodeInfo.OperatingSystem, node.Status.NodeInfo.Architecture) + Expect(err).NotTo(HaveOccurred(), "select the manifest for node %s", node.Name) + Expect(imageMirroringCheckImageID(pulled, []string{platformDigest, listDigest})).To(Succeed()) + }) } var _ = Describe("[sig-hypershift][Jira:Hypershift][Feature:ImageMirroring] Hosted Cluster Image Mirroring", Label("lifecycle", "image-mirroring", internal.InformingLabel), func() { diff --git a/test/e2e/v2/tests/hosted_cluster_image_mirroring_workload_helpers.go b/test/e2e/v2/tests/hosted_cluster_image_mirroring_workload_helpers.go new file mode 100644 index 000000000000..20154eb9048f --- /dev/null +++ b/test/e2e/v2/tests/hosted_cluster_image_mirroring_workload_helpers.go @@ -0,0 +1,336 @@ +//go:build e2ev2 + +package tests + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "regexp" + "strings" + "time" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + crclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + imageMirroringWorkloadContainer = "registry" + imageMirroringTrustBundleKey = "ca-bundle.crt" + imageMirroringManifestMaxBytes = 4 << 20 + imageMirroringManifestListType = "application/vnd.docker.distribution.manifest.list.v2+json" + imageMirroringOCIIndexType = "application/vnd.oci.image.index.v1+json" + imageMirroringHTTPClientTimeout = 30 * time.Second +) + +var imageMirroringDigestPattern = regexp.MustCompile(`^sha256:[a-f0-9]{64}$`) + +// imageMirroringVirtualMirror returns the host and repository path of the +// virtual source's mirror at the exact fixture Route host, as declared in the +// validated HC imageContentSources. +func imageMirroringVirtualMirror(expected []hyperv1.ImageContentSource, routeHost string) (string, string, error) { + sourceHost, _, _ := strings.Cut(imageMirroringVirtualSource, "/") + if !strings.HasSuffix(sourceHost, ".invalid") { + return "", "", fmt.Errorf("virtual source %q must use an unresolvable .invalid host", imageMirroringVirtualSource) + } + if strings.TrimSpace(routeHost) == "" { + return "", "", fmt.Errorf("mirror fixture Route host must not be empty") + } + for _, entry := range expected { + if entry.Source != imageMirroringVirtualSource { + continue + } + for _, mirror := range entry.Mirrors { + host, path, ok := strings.Cut(mirror, "/") + if ok && path != "" && host == routeHost { + return host, path, nil + } + } + return "", "", fmt.Errorf("HC source %q has no mirror at Route host %q: %v", entry.Source, routeHost, entry.Mirrors) + } + return "", "", fmt.Errorf("HC imageContentSources have no entry for virtual source %q: %+v", imageMirroringVirtualSource, expected) +} + +// imageMirroringDigestRef returns repository@digest, taking the digest from a +// by-digest image reference. +func imageMirroringDigestRef(repository, image string) (string, string, error) { + if strings.TrimSpace(repository) == "" || strings.Contains(repository, "@") { + return "", "", fmt.Errorf("repository %q must be nonblank and must not contain a digest", repository) + } + if lastSegment := repository[strings.LastIndex(repository, "/")+1:]; strings.Contains(lastSegment, ":") { + return "", "", fmt.Errorf("repository %q must not contain a tag", repository) + } + _, digest, ok := strings.Cut(image, "@") + if !ok || !imageMirroringDigestPattern.MatchString(digest) { + return "", "", fmt.Errorf("image %q must be referenced by a sha256 digest", image) + } + return repository + "@" + digest, digest, nil +} + +// imageMirroringImageIDDigest extracts the manifest digest from a container +// status imageID ("repo@sha256:..." or "docker-pullable://repo@sha256:..."). +// A bare image ID is rejected because it is not a manifest digest. +func imageMirroringImageIDDigest(imageID string) (string, error) { + ref := strings.TrimPrefix(imageID, "docker-pullable://") + at := strings.LastIndex(ref, "@") + if at <= 0 || !imageMirroringDigestPattern.MatchString(ref[at+1:]) { + return "", fmt.Errorf("imageID %q is not a repository@sha256 manifest reference", imageID) + } + return ref[at+1:], nil +} + +// imageMirroringTrustBundle reads the CA bundle referenced by the HC +// additionalTrustBundle from the HC namespace. +func imageMirroringTrustBundle(ctx context.Context, client crclient.Client, hc *hyperv1.HostedCluster) ([]byte, error) { + if hc == nil || hc.Spec.AdditionalTrustBundle == nil || strings.TrimSpace(hc.Spec.AdditionalTrustBundle.Name) == "" { + return nil, fmt.Errorf("HC must reference an additionalTrustBundle ConfigMap") + } + key := crclient.ObjectKey{Namespace: hc.Namespace, Name: hc.Spec.AdditionalTrustBundle.Name} + cm := &corev1.ConfigMap{} + if err := client.Get(ctx, key, cm); err != nil { + return nil, fmt.Errorf("read additionalTrustBundle ConfigMap %s: %w", key, err) + } + bundle := cm.Data[imageMirroringTrustBundleKey] + if strings.TrimSpace(bundle) == "" { + return nil, fmt.Errorf("additionalTrustBundle ConfigMap %s has an empty %s", key, imageMirroringTrustBundleKey) + } + return []byte(bundle), nil +} + +// imageMirroringHTTPClient returns an HTTPS client that trusts only caPEM. +func imageMirroringHTTPClient(caPEM []byte) (*http.Client, error) { + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + return nil, fmt.Errorf("trust bundle contains no PEM certificates") + } + return &http.Client{ + Timeout: imageMirroringHTTPClientTimeout, + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, + }, + }, nil +} + +// imageMirroringFetchManifest reads a manifest list or index by digest from the +// mirror's registry API. The body is returned unverified. +func imageMirroringFetchManifest(ctx context.Context, client *http.Client, host, repository, digest string) ([]byte, error) { + url := fmt.Sprintf("https://%s/v2/%s/manifests/%s", host, repository, digest) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("build request for %s: %w", url, err) + } + req.Header.Set("Accept", imageMirroringManifestListType+", "+imageMirroringOCIIndexType) + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("GET %s: %w", url, err) + } + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, imageMirroringManifestMaxBytes+1)) + if err != nil { + return nil, fmt.Errorf("read %s: %w", url, err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("GET %s: status %d: %.512s", url, resp.StatusCode, body) + } + if len(body) > imageMirroringManifestMaxBytes { + return nil, fmt.Errorf("GET %s: manifest exceeds %d bytes", url, imageMirroringManifestMaxBytes) + } + return body, nil +} + +// imageMirroringPlatformDigest verifies that body is the manifest list or +// index identified by listDigest and returns the digest of its single entry +// for os/architecture. +func imageMirroringPlatformDigest(body []byte, listDigest, os, architecture string) (string, error) { + sum := sha256.Sum256(body) + if got := "sha256:" + hex.EncodeToString(sum[:]); got != listDigest { + return "", fmt.Errorf("mirror manifest digest %s does not match requested digest %s", got, listDigest) + } + var list struct { + MediaType string `json:"mediaType"` + Manifests []struct { + Digest string `json:"digest"` + Platform *struct { + OS string `json:"os"` + Architecture string `json:"architecture"` + } `json:"platform"` + } `json:"manifests"` + } + if err := json.Unmarshal(body, &list); err != nil { + return "", fmt.Errorf("decode manifest list %s: %w", listDigest, err) + } + if (list.MediaType != imageMirroringManifestListType && list.MediaType != imageMirroringOCIIndexType && list.MediaType != "") || len(list.Manifests) == 0 { + return "", fmt.Errorf("manifest %s is not a nonempty manifest list or index (mediaType %q)", listDigest, list.MediaType) + } + var found []string + for _, m := range list.Manifests { + if m.Platform != nil && m.Platform.OS == os && m.Platform.Architecture == architecture { + found = append(found, m.Digest) + } + } + if len(found) != 1 { + return "", fmt.Errorf("manifest list %s has %d entries for %s/%s, expected exactly one", listDigest, len(found), os, architecture) + } + if !imageMirroringDigestPattern.MatchString(found[0]) { + return "", fmt.Errorf("manifest list %s has a malformed digest %q for %s/%s", listDigest, found[0], os, architecture) + } + return found[0], nil +} + +// imageMirroringReadyLinuxNodes counts Ready nodes labeled kubernetes.io/os=linux. +func imageMirroringReadyLinuxNodes(nodes []corev1.Node) int { + count := 0 + for _, node := range nodes { + if node.Labels[corev1.LabelOSStable] != "linux" { + continue + } + for _, cond := range node.Status.Conditions { + if cond.Type == corev1.NodeReady && cond.Status == corev1.ConditionTrue { + count++ + break + } + } + } + return count +} + +// imageMirroringWorkloadPod returns a restricted-compliant one-shot pod that +// always pulls image and runs the registry binary's version command. +func imageMirroringWorkloadPod(namespace, name, image string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + NodeSelector: map[string]string{corev1.LabelOSStable: "linux"}, + AutomountServiceAccountToken: ptr.To(false), + TerminationGracePeriodSeconds: ptr.To[int64](5), + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: ptr.To(true), + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + }, + Containers: []corev1.Container{{ + Name: imageMirroringWorkloadContainer, + Image: image, + ImagePullPolicy: corev1.PullAlways, + Command: []string{"/bin/registry", "--version"}, + Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("16Mi"), + }}, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + }, + }}, + }, + } +} + +// imageMirroringContainerStatus returns the workload container status, or nil +// when the pod is nil or has no status for that container. +func imageMirroringContainerStatus(pod *corev1.Pod) *corev1.ContainerStatus { + if pod == nil { + return nil + } + for i := range pod.Status.ContainerStatuses { + if pod.Status.ContainerStatuses[i].Name == imageMirroringWorkloadContainer { + return &pod.Status.ContainerStatuses[i] + } + } + return nil +} + +// imageMirroringPodSummary renders the pod state used in predicate reasons. +func imageMirroringPodSummary(pod *corev1.Pod) string { + if pod == nil { + return "pod " + } + summary := fmt.Sprintf("pod %s/%s phase=%q", pod.Namespace, pod.Name, pod.Status.Phase) + for _, cond := range pod.Status.Conditions { + if cond.Type == corev1.PodScheduled && cond.Status != corev1.ConditionTrue { + summary += fmt.Sprintf(" unscheduled=%s: %s", cond.Reason, cond.Message) + } + } + status := imageMirroringContainerStatus(pod) + switch { + case status == nil: + summary += fmt.Sprintf(" container %q has no status", imageMirroringWorkloadContainer) + case status.State.Waiting != nil: + summary += fmt.Sprintf(" waiting=%s: %s", status.State.Waiting.Reason, status.State.Waiting.Message) + case status.State.Terminated != nil: + summary += fmt.Sprintf(" terminated=%s exitCode=%d: %s", status.State.Terminated.Reason, status.State.Terminated.ExitCode, status.State.Terminated.Message) + case status.State.Running != nil: + summary += " running" + } + return summary +} + +// imageMirroringCheckPulledPod is done when the one-shot container exited 0 +// and the pod Succeeded. Failures that retries cannot fix are terminal errors. +func imageMirroringCheckPulledPod(pod *corev1.Pod) (bool, string, error) { + if pod == nil { + return false, "", fmt.Errorf("pod must not be nil") + } + summary := imageMirroringPodSummary(pod) + if pod.Status.Phase == corev1.PodFailed { + return false, "", fmt.Errorf("%s", summary) + } + status := imageMirroringContainerStatus(pod) + if status == nil { + return false, summary, nil + } + if waiting := status.State.Waiting; waiting != nil { + switch waiting.Reason { + case "CreateContainerConfigError", "CreateContainerError", "InvalidImageName", "ErrImageNeverPull": + return false, "", fmt.Errorf("%s", summary) + } + return false, summary, nil + } + if terminated := status.State.Terminated; terminated != nil { + if terminated.ExitCode != 0 { + return false, "", fmt.Errorf("%s", summary) + } + if pod.Status.Phase == corev1.PodSucceeded { + return true, summary, nil + } + } + return false, summary, nil +} + +// imageMirroringCheckImageID verifies that the container imageID digest is one +// of the accepted manifest digests. +func imageMirroringCheckImageID(pod *corev1.Pod, accepted []string) error { + if pod == nil { + return fmt.Errorf("pod must not be nil") + } + if len(accepted) == 0 { + return fmt.Errorf("no accepted digests for pod %s/%s", pod.Namespace, pod.Name) + } + status := imageMirroringContainerStatus(pod) + if status == nil { + return fmt.Errorf("pod %s/%s has no status for container %q", pod.Namespace, pod.Name, imageMirroringWorkloadContainer) + } + digest, err := imageMirroringImageIDDigest(status.ImageID) + if err != nil { + return fmt.Errorf("pod %s/%s: %w", pod.Namespace, pod.Name, err) + } + for _, want := range accepted { + if digest == want { + return nil + } + } + return fmt.Errorf("pod %s/%s imageID %q digest is not one of the mirror digests %v", pod.Namespace, pod.Name, status.ImageID, accepted) +} diff --git a/test/e2e/v2/tests/hosted_cluster_image_mirroring_workload_helpers_test.go b/test/e2e/v2/tests/hosted_cluster_image_mirroring_workload_helpers_test.go new file mode 100644 index 000000000000..c776729c68a9 --- /dev/null +++ b/test/e2e/v2/tests/hosted_cluster_image_mirroring_workload_helpers_test.go @@ -0,0 +1,503 @@ +//go:build e2ev2 + +package tests + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "encoding/hex" + "encoding/pem" + "math/big" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/test/e2e/v2/util/mirrorregistry" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +// imageMirroringTestManifestList is the verbatim manifest list of +// quay.io/libpod/registry:2.8.2 (skopeo inspect --raw). Its sha256 is the +// digest pinned in mirrorregistry.DefaultRegistryImage. +const imageMirroringTestManifestList = `{"manifests":[{"digest":"sha256:da1fbcd13a7ddc77d0d964a5c5c4cb707b5d440a028b0b42fe574b9e99077e27","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"architecture":"amd64","os":"linux"},"size":1363},{"digest":"sha256:ea32c4ea1ffa10d85c25b55e3e775a72013e685bf80f8da79ec5c5ddc4a4a63f","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"architecture":"arm","os":"linux","variant":"v6"},"size":1363},{"digest":"sha256:d72af0ec30bf3a5531f2d4dc4d0893f11b40cfd0a5dcbcbdcc19adde233847d2","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"architecture":"arm","os":"linux","variant":"v7"},"size":1363},{"digest":"sha256:fab2c6a4bb4acf3ee3ae611943e2596a73e9adb4da74432b1bcd686539f7f56f","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"architecture":"arm64","os":"linux","variant":"v8"},"size":1363},{"digest":"sha256:a353b4af55af90d0d30511d2191edb9da38f174390ce91e2329fa1839b6c2408","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"architecture":"ppc64le","os":"linux"},"size":1363},{"digest":"sha256:4cab339812e3bfa9e2242292f46ca2a99e804a4604d29f342f2b935381ebad3c","mediaType":"application\/vnd.docker.distribution.manifest.v2+json","platform":{"architecture":"s390x","os":"linux"},"size":1363}],"mediaType":"application\/vnd.docker.distribution.manifest.list.v2+json","schemaVersion":2}` + +const ( + imageMirroringTestListDigest = "sha256:e642e8604d305a3b82c8c1807b5df7a1a84cc650d57a60f9c5c2b78efec54b3f" + imageMirroringTestAMD64Digest = "sha256:da1fbcd13a7ddc77d0d964a5c5c4cb707b5d440a028b0b42fe574b9e99077e27" + imageMirroringTestARM64Digest = "sha256:fab2c6a4bb4acf3ee3ae611943e2596a73e9adb4da74432b1bcd686539f7f56f" +) + +func TestImageMirroringVirtualMirror(t *testing.T) { + tests := []struct { + name string + change func([]hyperv1.ImageContentSource) []hyperv1.ImageContentSource + routeHost string + wantHost string + wantPath string + wantErr string + }{ + {name: "When the virtual source has a mirror at the Route host, it should return its host and path", routeHost: "mirror.apps.ci.hypershift.dev", wantHost: "mirror.apps.ci.hypershift.dev", wantPath: "libpod/registry"}, + {name: "When the virtual source lists another mirror first, it should select the one at the Route host", routeHost: "mirror.apps.ci.hypershift.dev", wantHost: "mirror.apps.ci.hypershift.dev", wantPath: "libpod/registry", change: func(ics []hyperv1.ImageContentSource) []hyperv1.ImageContentSource { + ics[1].Mirrors = append([]string{"backup.apps.ci.hypershift.dev/libpod/registry"}, ics[1].Mirrors...) + return ics + }}, + {name: "When the Route host is empty, it should return an error", routeHost: " ", wantErr: "Route host"}, + {name: "When no mirror is at the Route host, it should return an error", routeHost: "other.apps.ci.hypershift.dev", wantErr: "no mirror at Route host"}, + {name: "When a mirror only shares the host prefix, it should return an error", routeHost: "mirror.apps.ci.hypershift.dev", wantErr: "no mirror at Route host", change: func(ics []hyperv1.ImageContentSource) []hyperv1.ImageContentSource { + ics[1].Mirrors = []string{"mirror.apps.ci.hypershift.dev.evil/libpod/registry"} + return ics + }}, + {name: "When the mirror has no repository path, it should return an error", routeHost: "mirror.apps.ci.hypershift.dev", wantErr: "no mirror at Route host", change: func(ics []hyperv1.ImageContentSource) []hyperv1.ImageContentSource { + ics[1].Mirrors = []string{"mirror.apps.ci.hypershift.dev"} + return ics + }}, + {name: "When the virtual source is missing, it should return an error", routeHost: "mirror.apps.ci.hypershift.dev", wantErr: "no entry for virtual source", change: func(ics []hyperv1.ImageContentSource) []hyperv1.ImageContentSource { + return ics[:1] + }}, + {name: "When the sources are nil, it should return an error", routeHost: "mirror.apps.ci.hypershift.dev", wantErr: "no entry for virtual source", change: func([]hyperv1.ImageContentSource) []hyperv1.ImageContentSource { + return nil + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ics := imageMirroringTestHC.DeepCopy().Spec.ImageContentSources + if tt.change != nil { + ics = tt.change(ics) + } + host, path, err := imageMirroringVirtualMirror(ics, tt.routeHost) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected error containing %q, got %v", tt.wantErr, err) + } + return + } + if err != nil || host != tt.wantHost || path != tt.wantPath { + t.Fatalf("got %q %q %v, want %q %q", host, path, err, tt.wantHost, tt.wantPath) + } + }) + } + t.Run("When the virtual source constant is inspected, it should use an unresolvable .invalid host", func(t *testing.T) { + host, _, _ := strings.Cut(imageMirroringVirtualSource, "/") + if !strings.HasSuffix(host, ".invalid") { + t.Fatalf("virtual source host %q must end in .invalid", host) + } + }) +} + +func TestImageMirroringDigestRef(t *testing.T) { + tests := []struct { + name string + repository string + image string + wantRef string + wantErr string + }{ + {name: "When the image is the pinned fixture image, it should build a by-digest reference on the repository", repository: imageMirroringVirtualSource, image: mirrorregistry.DefaultRegistryImage, wantRef: imageMirroringVirtualSource + "@" + imageMirroringTestListDigest}, + {name: "When the repository host has a port, it should accept it", repository: "registry.ci.local:5000/libpod/registry", image: mirrorregistry.DefaultRegistryImage, wantRef: "registry.ci.local:5000/libpod/registry@" + imageMirroringTestListDigest}, + {name: "When the image has no digest, it should return an error", repository: imageMirroringVirtualSource, image: "quay.io/libpod/registry:2.8.2", wantErr: "sha256 digest"}, + {name: "When the digest is malformed, it should return an error", repository: imageMirroringVirtualSource, image: "quay.io/libpod/registry@sha256:E642", wantErr: "sha256 digest"}, + {name: "When the repository is blank, it should return an error", repository: " ", image: mirrorregistry.DefaultRegistryImage, wantErr: "nonblank"}, + {name: "When the repository has a digest, it should return an error", repository: imageMirroringVirtualSource + "@" + imageMirroringTestListDigest, image: mirrorregistry.DefaultRegistryImage, wantErr: "digest"}, + {name: "When the repository has a tag, it should return an error", repository: imageMirroringVirtualSource + ":2.8.2", image: mirrorregistry.DefaultRegistryImage, wantErr: "tag"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ref, digest, err := imageMirroringDigestRef(tt.repository, tt.image) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected error containing %q, got %v", tt.wantErr, err) + } + return + } + if err != nil || ref != tt.wantRef || digest != imageMirroringTestListDigest { + t.Fatalf("got %q %q %v, want %q", ref, digest, err, tt.wantRef) + } + }) + } +} + +func TestImageMirroringImageIDDigest(t *testing.T) { + tests := []struct { + name string + imageID string + want string + wantErr bool + }{ + {name: "When the imageID is a CRI-O repository digest, it should return the digest", imageID: imageMirroringVirtualSource + "@" + imageMirroringTestAMD64Digest, want: imageMirroringTestAMD64Digest}, + {name: "When the imageID has the docker-pullable prefix, it should return the digest", imageID: "docker-pullable://" + imageMirroringVirtualSource + "@" + imageMirroringTestAMD64Digest, want: imageMirroringTestAMD64Digest}, + {name: "When the imageID is a bare image ID, it should return an error", imageID: imageMirroringTestAMD64Digest, wantErr: true}, + {name: "When the imageID is empty, it should return an error", imageID: "", wantErr: true}, + {name: "When the imageID digest is malformed, it should return an error", imageID: imageMirroringVirtualSource + "@sha256:abc", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := imageMirroringImageIDDigest(tt.imageID) + if tt.wantErr != (err != nil) || got != tt.want { + t.Fatalf("got %q %v, want %q (error %v)", got, err, tt.want, tt.wantErr) + } + }) + } +} + +func TestImageMirroringTrustBundle(t *testing.T) { + hc := &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: "clusters", Name: "image-mirroring"}, + Spec: hyperv1.HostedClusterSpec{AdditionalTrustBundle: &corev1.LocalObjectReference{Name: "user-ca-bundle"}}, + } + bundle := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Namespace: "clusters", Name: "user-ca-bundle"}, Data: map[string]string{"ca-bundle.crt": "-----BEGIN CERTIFICATE-----\n"}} + tests := []struct { + name string + hc *hyperv1.HostedCluster + objects []runtime.Object + want string + wantErr string + }{ + {name: "When the HC references a populated ConfigMap, it should return its CA bundle", hc: hc, objects: []runtime.Object{bundle}, want: "-----BEGIN CERTIFICATE-----\n"}, + {name: "When the HC is nil, it should return an error", hc: nil, wantErr: "additionalTrustBundle"}, + {name: "When the HC has no trust bundle, it should return an error", hc: &hyperv1.HostedCluster{}, wantErr: "additionalTrustBundle"}, + {name: "When the ConfigMap is missing, it should return an error", hc: hc, wantErr: "clusters/user-ca-bundle"}, + {name: "When the ConfigMap key is empty, it should return an error", hc: hc, objects: []runtime.Object{&corev1.ConfigMap{ObjectMeta: bundle.ObjectMeta}}, wantErr: "empty ca-bundle.crt"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := fake.NewClientBuilder().WithRuntimeObjects(tt.objects...).Build() + got, err := imageMirroringTrustBundle(t.Context(), client, tt.hc) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected error containing %q, got %v", tt.wantErr, err) + } + return + } + if err != nil || string(got) != tt.want { + t.Fatalf("got %q %v, want %q", got, err, tt.want) + } + }) + } +} + +func TestImageMirroringHTTPClient(t *testing.T) { + t.Run("When the trust bundle has no PEM certificate, it should return an error", func(t *testing.T) { + if _, err := imageMirroringHTTPClient([]byte("not a certificate")); err == nil { + t.Fatal("expected error") + } + }) + t.Run("When the trust bundle has a certificate, it should return a client with a timeout", func(t *testing.T) { + server := httptest.NewTLSServer(http.NotFoundHandler()) + defer server.Close() + client, err := imageMirroringHTTPClient(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})) + if err != nil || client.Timeout == 0 { + t.Fatalf("got %+v %v", client, err) + } + }) +} + +func TestImageMirroringFetchManifest(t *testing.T) { + const path = "/v2/mirror-source.hypershift-e2e.invalid/libpod/registry/manifests/" + imageMirroringTestListDigest + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != path { + http.Error(w, `{"errors":[{"code":"MANIFEST_UNKNOWN"}]}`, http.StatusNotFound) + return + } + accept := r.Header.Get("Accept") + if !strings.Contains(accept, imageMirroringManifestListType) || !strings.Contains(accept, imageMirroringOCIIndexType) { + http.Error(w, "missing list media types in Accept: "+accept, http.StatusNotAcceptable) + return + } + _, _ = w.Write([]byte(imageMirroringTestManifestList)) + }) + server := httptest.NewTLSServer(handler) + defer server.Close() + caOf := func(s *httptest.Server) []byte { + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: s.Certificate().Raw}) + } + host := strings.TrimPrefix(server.URL, "https://") + + tests := []struct { + name string + ca []byte + repository string + wantErr string + }{ + {name: "When the mirror serves the list with the trusted CA, it should return the exact body", ca: caOf(server), repository: imageMirroringVirtualSource}, + {name: "When the manifest is unknown, it should return the status and body", ca: caOf(server), repository: "libpod/other", wantErr: "status 404"}, + {name: "When the server certificate is not signed by the trust bundle, it should return a TLS error", ca: imageMirroringTestUnrelatedCA(t), repository: imageMirroringVirtualSource, wantErr: "certificate"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client, err := imageMirroringHTTPClient(tt.ca) + if err != nil { + t.Fatal(err) + } + body, err := imageMirroringFetchManifest(t.Context(), client, host, tt.repository, imageMirroringTestListDigest) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected error containing %q, got %v", tt.wantErr, err) + } + return + } + if err != nil || string(body) != imageMirroringTestManifestList { + t.Fatalf("got %d bytes, %v", len(body), err) + } + }) + } +} + +// imageMirroringTestUnrelatedCA returns a self-signed CA that did not sign +// the httptest certificate (all httptest servers share one certificate). +func imageMirroringTestUnrelatedCA(t *testing.T) []byte { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "unrelated-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) +} + +func TestImageMirroringPlatformDigest(t *testing.T) { + tests := []struct { + name string + body string + digest string + os string + arch string + want string + wantErr string + }{ + {name: "When the node is linux/amd64, it should return the amd64 manifest digest", body: imageMirroringTestManifestList, digest: imageMirroringTestListDigest, os: "linux", arch: "amd64", want: imageMirroringTestAMD64Digest}, + {name: "When the node is linux/arm64, it should return the arm64 manifest digest", body: imageMirroringTestManifestList, digest: imageMirroringTestListDigest, os: "linux", arch: "arm64", want: imageMirroringTestARM64Digest}, + {name: "When the body does not hash to the requested digest, it should return an error", body: imageMirroringTestManifestList + " ", digest: imageMirroringTestListDigest, os: "linux", arch: "amd64", wantErr: "does not match"}, + {name: "When several entries match the platform, it should return an error", body: imageMirroringTestManifestList, digest: imageMirroringTestListDigest, os: "linux", arch: "arm", wantErr: "2 entries"}, + {name: "When no entry matches the platform, it should return an error", body: imageMirroringTestManifestList, digest: imageMirroringTestListDigest, os: "windows", arch: "amd64", wantErr: "0 entries"}, + {name: "When the manifest is not a list, it should return an error", body: `{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json"}`, os: "linux", arch: "amd64", wantErr: "not a nonempty manifest list"}, + {name: "When the body is not JSON, it should return an error", body: "not json", os: "linux", arch: "amd64", wantErr: "decode"}, + {name: "When the matching entry digest is malformed, it should return an error", body: `{"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"digest":"sha256:abc","platform":{"os":"linux","architecture":"amd64"}}]}`, os: "linux", arch: "amd64", wantErr: "malformed"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + digest := tt.digest + if digest == "" { + digest = imageMirroringTestSHA256(tt.body) + } + got, err := imageMirroringPlatformDigest([]byte(tt.body), digest, tt.os, tt.arch) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected error containing %q, got %v", tt.wantErr, err) + } + return + } + if err != nil || got != tt.want { + t.Fatalf("got %q %v, want %q", got, err, tt.want) + } + }) + } + t.Run("When the test manifest list is hashed, it should match the pinned fixture image digest", func(t *testing.T) { + if _, digest, _ := strings.Cut(mirrorregistry.DefaultRegistryImage, "@"); imageMirroringTestSHA256(imageMirroringTestManifestList) != digest { + t.Fatalf("test manifest list does not hash to %s", digest) + } + }) +} + +func imageMirroringTestSHA256(body string) string { + sum := sha256.Sum256([]byte(body)) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func TestImageMirroringReadyLinuxNodes(t *testing.T) { + node := func(os string, ready corev1.ConditionStatus) corev1.Node { + return corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{corev1.LabelOSStable: os}}, + Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: ready}}}, + } + } + tests := []struct { + name string + nodes []corev1.Node + want int + }{ + {name: "When nodes are Ready linux nodes, it should count them", nodes: []corev1.Node{node("linux", corev1.ConditionTrue), node("linux", corev1.ConditionTrue)}, want: 2}, + {name: "When a linux node is not Ready, it should not count it", nodes: []corev1.Node{node("linux", corev1.ConditionFalse), node("linux", corev1.ConditionUnknown)}, want: 0}, + {name: "When a Ready node is not linux, it should not count it", nodes: []corev1.Node{node("windows", corev1.ConditionTrue), {}}, want: 0}, + {name: "When there are no nodes, it should return zero", want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := imageMirroringReadyLinuxNodes(tt.nodes); got != tt.want { + t.Fatalf("got %d, want %d", got, tt.want) + } + }) + } +} + +func TestImageMirroringWorkloadPod(t *testing.T) { + t.Run("When a workload pod is built, it should be a restricted one-shot pod that always pulls the image", func(t *testing.T) { + pod := imageMirroringWorkloadPod("e2e-image-mirroring-x", "by-digest", imageMirroringVirtualSource+"@"+imageMirroringTestListDigest) + if pod.Namespace != "e2e-image-mirroring-x" || pod.Name != "by-digest" || pod.Spec.RestartPolicy != corev1.RestartPolicyNever { + t.Fatalf("unexpected identity or restart policy: %+v", pod.ObjectMeta) + } + if pod.Spec.NodeSelector[corev1.LabelOSStable] != "linux" || pod.Spec.AutomountServiceAccountToken == nil || *pod.Spec.AutomountServiceAccountToken { + t.Fatalf("expected linux node selector and no service account token: %+v", pod.Spec) + } + psc := pod.Spec.SecurityContext + if psc == nil || psc.RunAsNonRoot == nil || !*psc.RunAsNonRoot || psc.RunAsUser != nil || psc.SeccompProfile == nil || psc.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault { + t.Fatalf("pod security context is not restricted: %+v", psc) + } + if len(pod.Spec.Containers) != 1 { + t.Fatalf("expected one container, got %d", len(pod.Spec.Containers)) + } + c := pod.Spec.Containers[0] + if c.Name != imageMirroringWorkloadContainer || c.Image != imageMirroringVirtualSource+"@"+imageMirroringTestListDigest || c.ImagePullPolicy != corev1.PullAlways { + t.Fatalf("unexpected container image settings: %+v", c) + } + if !reflect.DeepEqual(c.Command, []string{"/bin/registry", "--version"}) { + t.Fatalf("unexpected command %v", c.Command) + } + csc := c.SecurityContext + if csc == nil || csc.AllowPrivilegeEscalation == nil || *csc.AllowPrivilegeEscalation || csc.Capabilities == nil || !reflect.DeepEqual(csc.Capabilities.Drop, []corev1.Capability{"ALL"}) { + t.Fatalf("container security context is not restricted: %+v", csc) + } + }) +} + +func imageMirroringTestPod(phase corev1.PodPhase, state *corev1.ContainerState) *corev1.Pod { + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "e2e-image-mirroring-x", Name: "pod"}, Status: corev1.PodStatus{Phase: phase}} + if state != nil { + pod.Status.ContainerStatuses = []corev1.ContainerStatus{{Name: imageMirroringWorkloadContainer, State: *state}} + } + return pod +} + +func imageMirroringWaitingState(reason, message string) *corev1.ContainerState { + return &corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: reason, Message: message}} +} + +func imageMirroringTerminatedState(exitCode int32) *corev1.ContainerState { + return &corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: "Completed", ExitCode: exitCode}} +} + +func imageMirroringRunningState() *corev1.ContainerState { + return &corev1.ContainerState{Running: &corev1.ContainerStateRunning{}} +} + +func TestImageMirroringCheckPulledPod(t *testing.T) { + unschedulable := imageMirroringTestPod(corev1.PodPending, nil) + unschedulable.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: "Unschedulable", Message: "0/1 nodes are available"}} + otherContainer := imageMirroringTestPod(corev1.PodPending, nil) + otherContainer.Status.ContainerStatuses = []corev1.ContainerStatus{{Name: "other", State: *imageMirroringTerminatedState(0)}} + + tests := []struct { + name string + pod *corev1.Pod + wantDone bool + wantReason string + wantErr string + }{ + {name: "When the container exited 0 and the pod Succeeded, it should be done", pod: imageMirroringTestPod(corev1.PodSucceeded, imageMirroringTerminatedState(0)), wantDone: true}, + {name: "When the pod is Pending without container statuses, it should keep polling", pod: imageMirroringTestPod(corev1.PodPending, nil), wantReason: "has no status"}, + {name: "When the pod is unschedulable, it should keep polling and report the reason", pod: unschedulable, wantReason: "Unschedulable: 0/1 nodes are available"}, + {name: "When only another container has a status, it should keep polling", pod: otherContainer, wantReason: "has no status"}, + {name: "When the container is creating, it should keep polling", pod: imageMirroringTestPod(corev1.PodPending, imageMirroringWaitingState("ContainerCreating", "")), wantReason: "ContainerCreating"}, + {name: "When the image pull fails, it should keep polling and report the message", pod: imageMirroringTestPod(corev1.PodPending, imageMirroringWaitingState("ErrImagePull", "no such host")), wantReason: "ErrImagePull: no such host"}, + {name: "When the image pull backs off, it should keep polling", pod: imageMirroringTestPod(corev1.PodPending, imageMirroringWaitingState("ImagePullBackOff", "")), wantReason: "ImagePullBackOff"}, + {name: "When the container is running, it should keep polling", pod: imageMirroringTestPod(corev1.PodRunning, imageMirroringRunningState()), wantReason: "running"}, + {name: "When the container exited 0 but the pod has not Succeeded yet, it should keep polling", pod: imageMirroringTestPod(corev1.PodRunning, imageMirroringTerminatedState(0)), wantReason: "exitCode=0"}, + {name: "When the container config cannot be created, it should fail terminally", pod: imageMirroringTestPod(corev1.PodPending, imageMirroringWaitingState("CreateContainerConfigError", "container has runAsNonRoot and image will run as root")), wantErr: "CreateContainerConfigError"}, + {name: "When the image name is invalid, it should fail terminally", pod: imageMirroringTestPod(corev1.PodPending, imageMirroringWaitingState("InvalidImageName", "")), wantErr: "InvalidImageName"}, + {name: "When the container exits non-zero, it should fail terminally", pod: imageMirroringTestPod(corev1.PodRunning, imageMirroringTerminatedState(2)), wantErr: "exitCode=2"}, + {name: "When the pod Failed, it should fail terminally", pod: imageMirroringTestPod(corev1.PodFailed, nil), wantErr: "Failed"}, + {name: "When the pod is nil, it should fail terminally", pod: nil, wantErr: "nil"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + done, reason, err := imageMirroringCheckPulledPod(tt.pod) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) || done { + t.Fatalf("expected terminal error containing %q, got %v %v", tt.wantErr, done, err) + } + return + } + if err != nil || done != tt.wantDone || !strings.Contains(reason, tt.wantReason) || !strings.Contains(reason, "e2e-image-mirroring-x/pod") { + t.Fatalf("got %v %q %v, want %v containing %q", done, reason, err, tt.wantDone, tt.wantReason) + } + }) + } +} + +func TestImageMirroringPodState(t *testing.T) { + t.Run("When the pod is nil, it should not panic and report no container status", func(t *testing.T) { + if got := imageMirroringContainerStatus(nil); got != nil { + t.Fatalf("expected nil status, got %+v", got) + } + if got := imageMirroringPodSummary(nil); !strings.Contains(got, "") { + t.Fatalf("expected a nil pod summary, got %q", got) + } + }) + t.Run("When the workload container has a status, it should return that status", func(t *testing.T) { + pod := imageMirroringTestPod(corev1.PodRunning, imageMirroringRunningState()) + if got := imageMirroringContainerStatus(pod); got == nil || got.Name != imageMirroringWorkloadContainer { + t.Fatalf("expected the %q status, got %+v", imageMirroringWorkloadContainer, got) + } + }) +} + +func TestImageMirroringCheckImageID(t *testing.T) { + withImageID := func(imageID string) *corev1.Pod { + pod := imageMirroringTestPod(corev1.PodSucceeded, imageMirroringTerminatedState(0)) + pod.Status.ContainerStatuses[0].ImageID = imageID + return pod + } + accepted := []string{imageMirroringTestAMD64Digest, imageMirroringTestListDigest} + tests := []struct { + name string + pod *corev1.Pod + accepted []string + wantErr string + }{ + {name: "When the imageID has the per-platform digest, it should succeed", pod: withImageID(imageMirroringVirtualSource + "@" + imageMirroringTestAMD64Digest), accepted: accepted}, + {name: "When the imageID has the manifest list digest, it should succeed", pod: withImageID(imageMirroringVirtualSource + "@" + imageMirroringTestListDigest), accepted: accepted}, + {name: "When the imageID has another platform digest, it should return an error", pod: withImageID(imageMirroringVirtualSource + "@" + imageMirroringTestARM64Digest), accepted: accepted, wantErr: "not one of the mirror digests"}, + {name: "When the imageID is a bare image ID, it should return an error", pod: withImageID(imageMirroringTestAMD64Digest), accepted: accepted, wantErr: "manifest reference"}, + {name: "When the container has no status, it should return an error", pod: imageMirroringTestPod(corev1.PodSucceeded, nil), accepted: accepted, wantErr: "no status"}, + {name: "When no digests are accepted, it should return an error", pod: withImageID(imageMirroringVirtualSource + "@" + imageMirroringTestAMD64Digest), wantErr: "no accepted digests"}, + {name: "When the pod is nil, it should return an error", pod: nil, accepted: accepted, wantErr: "nil"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := imageMirroringCheckImageID(tt.pod, tt.accepted) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected error containing %q, got %v", tt.wantErr, err) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + } +} From c54798ae9cf92831126988691b0c4d8bcf610b0d Mon Sep 17 00:00:00 2001 From: Alessandro Rossi Date: Wed, 28 Oct 2026 10:45:46 +0100 Subject: [PATCH 2/2] build: run image mirroring e2e v2 helper unit tests in test-e2ev2-unit The pure helper tests in test/e2e/v2/tests were not run by any CI job: the GHA unit shards do not include that package, because running it whole would start the Ginkgo suite, which needs a hosted cluster. Add TestImageMirroring.* to the existing test-e2ev2-unit -run filter. make test depends on that target, so ci/prow/unit now runs them. Signed-off-by: Alessandro Rossi Assisted-by: LLM --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index af3905b76d35..721cbdcfdb9a 100644 --- a/Makefile +++ b/Makefile @@ -443,7 +443,7 @@ test: generate test-e2ev2-unit .PHONY: test-e2ev2-unit test-e2ev2-unit: $(GO) test $(GO_TEST_FLAGS) -tags=e2ev2 -count=1 -timeout=10m ./test/e2e/v2/internal ./test/e2e/v2/cmd/run-tests - $(GO) test $(GO_TEST_FLAGS) -tags=e2ev2 -count=1 -timeout=10m -run '^(TestComponentSupportsTLSConfig|TestVerifyIntermediateTLSConnectivityAfterDowngrade)$$' ./test/e2e/v2/tests + $(GO) test $(GO_TEST_FLAGS) -tags=e2ev2 -count=1 -timeout=10m -run '^(TestComponentSupportsTLSConfig|TestVerifyIntermediateTLSConnectivityAfterDowngrade|TestImageMirroring.*)$$' ./test/e2e/v2/tests # Run tests only for Go packages with changes relative to PULL_BASE_SHA. # Skips entirely if no .go files changed. No generate dependency (verify-quick handles it).