CNTRLPLANE-3928: e2e: verify workload pulls through the image mirror

Makefile

@@ -443,7 +443,7 @@ test: generate test-e2ev2-unit
 .PHONY: test-e2ev2-unit
 test-e2ev2-unit:
 	$(GO) test $(GO_TEST_FLAGS) -tags=e2ev2 -count=1 -timeout=10m ./test/e2e/v2/internal ./test/e2e/v2/cmd/run-tests
-	$(GO) test $(GO_TEST_FLAGS) -tags=e2ev2 -count=1 -timeout=10m -run '^(TestComponentSupportsTLSConfig|TestVerifyIntermediateTLSConnectivityAfterDowngrade)$$' ./test/e2e/v2/tests
+	$(GO) test $(GO_TEST_FLAGS) -tags=e2ev2 -count=1 -timeout=10m -run '^(TestComponentSupportsTLSConfig|TestVerifyIntermediateTLSConnectivityAfterDowngrade|TestImageMirroring.*)$$' ./test/e2e/v2/tests
 
 # Run tests only for Go packages with changes relative to PULL_BASE_SHA.
 # Skips entirely if no .go files changed. No generate dependency (verify-quick handles it).

test/e2e/v2/tests/hosted_cluster_image_mirroring_helpers.go

@@ -9,11 +9,13 @@ import (
 	"strings"
 
 	hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
+	"github.com/openshift/hypershift/test/e2e/v2/util/mirrorregistry"
 
 	configv1 "github.com/openshift/api/config/v1"
 	routev1 "github.com/openshift/api/route/v1"
 
 	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
 
 	crclient "sigs.k8s.io/controller-runtime/pkg/client"
@@ -24,6 +26,11 @@ const (
 	imageMirroringIgnitionName = "ignition-config-40-image-content-source"
 	imageMirroringWorkerLabel  = "machineconfiguration.openshift.io/role"
 	imageMirroringCoreLabel    = "hypershift.openshift.io/core-ignition-config"
+
+	// imageMirroringVirtualSource is the fixture source repository that only
+	// exists on the mirror: its ".invalid" host never resolves (RFC 2606), so a
+	// successful pull from it can only have been served by an IDMS mirror.
+	imageMirroringVirtualSource = "mirror-source.hypershift-e2e.invalid/libpod/registry"
 )
 
 // imageMirroringConfigured reports whether the HC opted into image mirroring
@@ -33,6 +40,29 @@ func imageMirroringConfigured(hc *hyperv1.HostedCluster, routeFound bool) bool {
 	return hc != nil && len(hc.Spec.ImageContentSources) > 0 && routeFound
 }
 
+// imageMirroringLoadSetup reads the mirror fixture Route from the management
+// cluster and validates the HC image-mirroring setup against it. It returns
+// configured=false and no error when the HC has no imageContentSources or the
+// Route does not exist, so the caller can skip. A Route read error other than
+// NotFound, or an invalid setup once both are present, is returned as an error
+// so the caller fails instead of skipping.
+func imageMirroringLoadSetup(ctx context.Context, client crclient.Client, hc *hyperv1.HostedCluster) (bool, []hyperv1.ImageContentSource, string, error) {
+	key := crclient.ObjectKey{Namespace: mirrorregistry.DefaultNamespace, Name: mirrorregistry.DefaultName}
+	route := &routev1.Route{}
+	err := client.Get(ctx, key, route)
+	if err != nil && !apierrors.IsNotFound(err) {
+		return false, nil, "", fmt.Errorf("read mirror fixture Route %s: %w", key, err)
+	}
+	if !imageMirroringConfigured(hc, err == nil) {
+		return false, nil, "", nil
+	}
+	expected, err := imageMirroringExpected(hc, route)
+	if err != nil {
+		return false, nil, "", err
+	}
+	return true, expected, route.Spec.Host, nil
+}
+
 // Expectations come only from a validated snapshot of HC input, never from a
 // producer conversion or either output. Mirror priority and multiplicity matter.
 func imageMirroringExpected(hc *hyperv1.HostedCluster, route *routev1.Route) ([]hyperv1.ImageContentSource, error) {
@@ -58,7 +88,7 @@ func imageMirroringExpected(hc *hyperv1.HostedCluster, route *routev1.Route) ([]
 	if route == nil || strings.TrimSpace(route.Spec.Host) == "" {
 		return nil, fmt.Errorf("mirror fixture Route has an empty host")
 	}
-	for _, source := range []string{"quay.io/libpod/registry", "mirror-source.hypershift-e2e.invalid/libpod/registry"} {
+	for _, source := range []string{"quay.io/libpod/registry", imageMirroringVirtualSource} {
 		found := false
 		for _, entry := range expected {
 			if entry.Source != source {

test/e2e/v2/tests/hosted_cluster_image_mirroring_helpers_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 
 	hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
+	"github.com/openshift/hypershift/test/e2e/v2/util/mirrorregistry"
 
 	configv1 "github.com/openshift/api/config/v1"
 	routev1 "github.com/openshift/api/route/v1"
@@ -150,6 +151,70 @@ func TestImageMirroringConfigured(t *testing.T) {
 	}
 }
 
+func TestImageMirroringLoadSetup(t *testing.T) {
+	sentinel := errors.New("API unavailable")
+	fixtureRoute := imageMirroringTestRoute.DeepCopy()
+	fixtureRoute.Namespace, fixtureRoute.Name = mirrorregistry.DefaultNamespace, mirrorregistry.DefaultName
+	otherRoute := fixtureRoute.DeepCopy()
+	otherRoute.Namespace = "other"
+	invalid := imageMirroringTestHC.DeepCopy()
+	invalid.Spec.ImageContentSources = invalid.Spec.ImageContentSources[:1]
+	for _, tt := range []struct {
+		name           string
+		hc             *hyperv1.HostedCluster
+		objects        []crclient.Object
+		failure        error
+		wantConfigured bool
+		wantErr        string
+	}{
+		{name: "When the HC has imageContentSources and the fixture Route exists, it should return the validated setup", hc: imageMirroringTestHC, objects: []crclient.Object{fixtureRoute}, wantConfigured: true},
+		{name: "When the fixture Route does not exist, it should report not configured without error so the spec skips", hc: imageMirroringTestHC},
+		{name: "When only a Route in another namespace exists, it should report not configured without error", hc: imageMirroringTestHC, objects: []crclient.Object{otherRoute}},
+		{name: "When the HC has no imageContentSources, it should report not configured without error so the spec skips", hc: &hyperv1.HostedCluster{}, objects: []crclient.Object{fixtureRoute}},
+		{name: "When the Route read fails, it should return the error so the spec fails instead of skipping", hc: imageMirroringTestHC, failure: sentinel, wantErr: "API unavailable"},
+		{name: "When the setup is present but invalid, it should return the error so the spec fails", hc: invalid, objects: []crclient.Object{fixtureRoute}, wantErr: "mirror-source"},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			scheme := runtime.NewScheme()
+			if err := routev1.Install(scheme); err != nil {
+				t.Fatal(err)
+			}
+			ctx := context.WithValue(t.Context(), imageMirroringContextKey{}, "spec context")
+			client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tt.objects...).WithInterceptorFuncs(interceptor.Funcs{Get: func(got context.Context, c crclient.WithWatch, key crclient.ObjectKey, o crclient.Object, opts ...crclient.GetOption) error {
+				if got != ctx || key != (crclient.ObjectKey{Namespace: mirrorregistry.DefaultNamespace, Name: mirrorregistry.DefaultName}) {
+					t.Fatalf("wrong context or key: %v", key)
+				}
+				if tt.failure != nil {
+					return tt.failure
+				}
+				return c.Get(got, key, o, opts...)
+			}}).Build()
+			configured, expected, routeHost, err := imageMirroringLoadSetup(ctx, client, tt.hc)
+			if tt.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tt.wantErr) || configured {
+					t.Fatalf("expected error containing %q, got configured=%v err=%v", tt.wantErr, configured, err)
+				}
+				if tt.failure != nil && !errors.Is(err, tt.failure) {
+					t.Fatalf("lost API error: %v", err)
+				}
+				return
+			}
+			if err != nil || configured != tt.wantConfigured {
+				t.Fatalf("got configured=%v err=%v, want configured=%v", configured, err, tt.wantConfigured)
+			}
+			if !tt.wantConfigured {
+				if expected != nil || routeHost != "" {
+					t.Fatalf("unconfigured setup returned data: %+v %q", expected, routeHost)
+				}
+				return
+			}
+			if !reflect.DeepEqual(expected, tt.hc.Spec.ImageContentSources) || routeHost != fixtureRoute.Spec.Host {
+				t.Fatalf("got %+v %q", expected, routeHost)
+			}
+		})
+	}
+}
+
 func TestImageMirroringCheckMappings(t *testing.T) {
 	tests := []struct {
 		name   string

test/e2e/v2/tests/hosted_cluster_image_mirroring_test.go

@@ -14,40 +14,40 @@ import (
 	"github.com/openshift/hypershift/test/e2e/v2/internal"
 	"github.com/openshift/hypershift/test/e2e/v2/util/mirrorregistry"
 
-	routev1 "github.com/openshift/api/route/v1"
-
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	crclient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 // imageMirroringSetupOrSkip skips when the HC has no imageContentSources or the
 // mirror fixture Route does not exist. Once both are present, the setup must be
-// valid: malformed, duplicate, or non-fixture mappings fail the spec.
-func imageMirroringSetupOrSkip(tc *internal.TestContext) (*hyperv1.HostedCluster, []hyperv1.ImageContentSource) {
+// valid: malformed, duplicate, or non-fixture mappings fail the spec, as do API
+// errors. It also returns the fixture Route host. The decision logic lives in
+// imageMirroringLoadSetup; this wrapper only asserts and skips.
+func imageMirroringSetupOrSkip(g Gomega, tc *internal.TestContext) (*hyperv1.HostedCluster, []hyperv1.ImageContentSource, string) {
 	GinkgoHelper()
-	Expect(tc).NotTo(BeNil(), "test context must be initialized")
+	g.Expect(tc).NotTo(BeNil(), "test context must be initialized")
 	hc, err := tc.GetHostedCluster()
-	Expect(err).NotTo(HaveOccurred(), "read HC %s/%s", tc.ClusterNamespace, tc.ClusterName)
-	route := &routev1.Route{}
-	err = tc.MgmtClient.Get(tc.Context, crclient.ObjectKey{Namespace: mirrorregistry.DefaultNamespace, Name: mirrorregistry.DefaultName}, route)
-	if err != nil && !apierrors.IsNotFound(err) {
-		Expect(err).NotTo(HaveOccurred(), "read mirror fixture Route %s/%s", mirrorregistry.DefaultNamespace, mirrorregistry.DefaultName)
-	}
-	if !imageMirroringConfigured(hc, err == nil) {
+	g.Expect(err).NotTo(HaveOccurred(), "read HC %s/%s", tc.ClusterNamespace, tc.ClusterName)
+	g.Expect(hc).NotTo(BeNil(), "HC %s/%s", tc.ClusterNamespace, tc.ClusterName)
+	configured, expected, routeHost, err := imageMirroringLoadSetup(tc.Context, tc.MgmtClient, hc)
+	g.Expect(err).NotTo(HaveOccurred(), "HC %s/%s image-mirroring setup", hc.Namespace, hc.Name)
+	if !configured {
 		Skip("image mirroring is not configured on this hosted cluster")
 	}
-	expected, err := imageMirroringExpected(hc, route)
-	Expect(err).NotTo(HaveOccurred(), "HC %s/%s image-mirroring setup", hc.Namespace, hc.Name)
-	return hc, expected
+	return hc, expected, routeHost
 }
 
-// RegisterHostedClusterImageMirroringTests registers independent, read-only
-// lifecycle checks for the HCCO IDMS and CPO ignition propagation paths.
+// RegisterHostedClusterImageMirroringTests registers independent lifecycle
+// checks for the HCCO IDMS and CPO ignition propagation paths and for workload
+// pulls through the mirror on data plane nodes. They never mutate the HC; the
+// workload check only creates and deletes its own namespace and pods.
 func RegisterHostedClusterImageMirroringTests(getTestCtx internal.TestContextGetter) {
 	It("should propagate imageContentSources to the hosted cluster IDMS with the worker role", func() {
 		tc := getTestCtx()
-		hc, expected := imageMirroringSetupOrSkip(tc)
+		hc, expected, _ := imageMirroringSetupOrSkip(Default, tc)
 
 		hostedClient, err := tc.GetHostedClusterClientViaPortForward(hc)
 		Expect(err).NotTo(HaveOccurred(), "open hosted client for HC %s/%s", hc.Namespace, hc.Name)
@@ -66,7 +66,7 @@ func RegisterHostedClusterImageMirroringTests(getTestCtx internal.TestContextGet
 
 	It("should propagate imageContentSources independently to the control plane ignition ConfigMap", func() {
 		tc := getTestCtx()
-		hc, expected := imageMirroringSetupOrSkip(tc)
+		hc, expected, _ := imageMirroringSetupOrSkip(Default, tc)
 		Expect(tc.ControlPlaneNamespace).NotTo(BeEmpty(), "HC %s/%s must have a control plane namespace", hc.Namespace, hc.Name)
 
 		Eventually(func(g Gomega, ctx context.Context) {
@@ -80,6 +80,72 @@ func RegisterHostedClusterImageMirroringTests(getTestCtx internal.TestContextGet
 		}).WithContext(tc.Context).WithTimeout(10*time.Minute).WithPolling(3*time.Second).Should(Succeed(),
 			"HC %s/%s ConfigMap %s/%s decoded IDMS mappings and labels", hc.Namespace, hc.Name, tc.ControlPlaneNamespace, imageMirroringIgnitionName)
 	})
+	It("should pull a workload image by digest through the mirror when the source is unreachable", func() {
+		tc := getTestCtx()
+		hc, expected, routeHost := imageMirroringSetupOrSkip(Default, tc)
+
+		// Expectations come from the HC spec and the pinned input image, never
+		// from the fixture result.
+		mirrorHost, mirrorRepository, err := imageMirroringVirtualMirror(expected, routeHost)
+		Expect(err).NotTo(HaveOccurred(), "HC %s/%s virtual source mirror", hc.Namespace, hc.Name)
+		digestRef, listDigest, err := imageMirroringDigestRef(imageMirroringVirtualSource, mirrorregistry.DefaultRegistryImage)
+		Expect(err).NotTo(HaveOccurred(), "build by-digest reference")
+
+		// CRI-O reports the per-platform manifest digest for a manifest-list
+		// pull, so read the list from the mirror and verify it by content.
+		caPEM, err := imageMirroringTrustBundle(tc.Context, tc.MgmtClient, hc)
+		Expect(err).NotTo(HaveOccurred(), "HC %s/%s trust bundle", hc.Namespace, hc.Name)
+		httpClient, err := imageMirroringHTTPClient(caPEM)
+		Expect(err).NotTo(HaveOccurred(), "HC %s/%s trust bundle client", hc.Namespace, hc.Name)
+		manifestList, err := imageMirroringFetchManifest(tc.Context, httpClient, mirrorHost, mirrorRepository, listDigest)
+		Expect(err).NotTo(HaveOccurred(), "read manifest list %s from mirror %s/%s", listDigest, mirrorHost, mirrorRepository)
+
+		hostedClient, err := tc.GetHostedClusterClientViaPortForward(hc)
+		Expect(err).NotTo(HaveOccurred(), "open hosted client for HC %s/%s", hc.Namespace, hc.Name)
+		DeferCleanup(hostedClient.Close)
+
+		nodes := &corev1.NodeList{}
+		Expect(hostedClient.List(tc.Context, nodes)).To(Succeed(), "list nodes of HC %s/%s", hc.Namespace, hc.Name)
+		Expect(imageMirroringReadyLinuxNodes(nodes.Items)).To(BeNumerically(">", 0), "HC %s/%s must have a Ready linux node", hc.Namespace, hc.Name)
+
+		namespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "e2e-image-mirroring-"}}
+		Expect(hostedClient.Create(tc.Context, namespace)).To(Succeed(), "create workload namespace in HC %s/%s", hc.Namespace, hc.Name)
+		DeferCleanup(func(ctx context.Context) {
+			if err := hostedClient.Delete(ctx, namespace); err != nil && !apierrors.IsNotFound(err) {
+				Expect(err).NotTo(HaveOccurred(), "cleanup: delete namespace %s", namespace.Name)
+			}
+		})
+
+		byDigest := imageMirroringWorkloadPod(namespace.Name, "by-digest", digestRef)
+		Expect(hostedClient.Create(tc.Context, byDigest)).To(Succeed(), "create pod %s/%s", byDigest.Namespace, byDigest.Name)
+		DeferCleanup(func(ctx context.Context) {
+			if err := hostedClient.Delete(ctx, byDigest); err != nil && !apierrors.IsNotFound(err) {
+				Expect(err).NotTo(HaveOccurred(), "cleanup: delete pod %s/%s", byDigest.Namespace, byDigest.Name)
+			}
+		})
+
+		// GET errors and unfinished states keep polling; a check error (a failure
+		// that retries cannot fix) stops at once.
+		waitObjective := fmt.Sprintf("pod %s/%s (%s) to pull through the mirror and succeed", byDigest.Namespace, byDigest.Name, digestRef)
+		Eventually(func(g Gomega, ctx context.Context) {
+			current := &corev1.Pod{}
+			g.Expect(hostedClient.Get(ctx, crclient.ObjectKeyFromObject(byDigest), current)).To(Succeed(), "get pod %s/%s", byDigest.Namespace, byDigest.Name)
+			done, reason, err := imageMirroringCheckPulledPod(current)
+			if err != nil {
+				StopTrying(waitObjective).Wrap(err).Now()
+			}
+			g.Expect(done).To(BeTrue(), reason)
+		}).WithContext(tc.Context).WithTimeout(10*time.Minute).WithPolling(5*time.Second).Should(Succeed(), waitObjective)
+
+		pulled := &corev1.Pod{}
+		Expect(hostedClient.Get(tc.Context, crclient.ObjectKeyFromObject(byDigest), pulled)).To(Succeed(), "read pod %s/%s", byDigest.Namespace, byDigest.Name)
+		Expect(pulled.Spec.NodeName).NotTo(BeEmpty(), "pod %s/%s must be bound to a node", pulled.Namespace, pulled.Name)
+		node := &corev1.Node{}
+		Expect(hostedClient.Get(tc.Context, crclient.ObjectKey{Name: pulled.Spec.NodeName}, node)).To(Succeed(), "read node %s", pulled.Spec.NodeName)
+		platformDigest, err := imageMirroringPlatformDigest(manifestList, listDigest, node.Status.NodeInfo.OperatingSystem, node.Status.NodeInfo.Architecture)
+		Expect(err).NotTo(HaveOccurred(), "select the manifest for node %s", node.Name)
+		Expect(imageMirroringCheckImageID(pulled, []string{platformDigest, listDigest})).To(Succeed())
+	})
 }
 
 var _ = Describe("[sig-hypershift][Jira:Hypershift][Feature:ImageMirroring] Hosted Cluster Image Mirroring", Label("lifecycle", "image-mirroring", internal.InformingLabel), func() {