From a4623a5f4696e85e72ebd15436e0e7bf05216f60 Mon Sep 17 00:00:00 2001 From: David Fridrich Date: Tue, 13 Oct 2026 21:13:03 +0200 Subject: [PATCH] e2e: run the functional suite against a real OCP cluster Add hack/openshift/ to run the e2e suite against an existing OpenShift cluster an engineer is oc-login'd to: - prepare-ocp-cluster.sh: idempotent, env-gated setup. Installs Gitea (+ kube-public/gitea-endpoint ConfigMap), Red Hat OpenShift Pipelines, and (optionally) CMA for the keda deployer. No registry is deployed: the suite pushes to ttl.sh with on-cluster builds (FUNC_REMOTE=true). - run-e2e-tests.sh: sets REGISTRY to a fresh ttl.sh namespace, DEFAULT_DEPLOYER=raw, FUNC_REMOTE=true and FUNC_MANAGEMENT_DISABLED=true, and execs make test-e2e. - Makefile: pass E2E_SKIP and STALE_FUNCTION_IMAGE through to ginkgo. The default E2E_SKIP drops specs that cannot work under FUNC_REMOTE=true (details in run-e2e-tests.sh): the SSH-repository specs (need an exposed SSH endpoint), Middleware Update and registry-auth-secret (their stale-CLI seed premise breaks remotely), and the monorepo subdirectory spec (the remote seed deploy leaves the function name out of the committed func.yaml, so the operator's describe fails). The middleware-rebuild spec self-skips while STALE_FUNCTION_IMAGE is unset. --- hack/openshift/manifests/cma.yaml | 27 ++++++ hack/openshift/manifests/gitea.yaml | 118 +++++++++++++++++++++++ hack/openshift/manifests/pipelines.yaml | 15 +++ hack/openshift/prepare-ocp-cluster.sh | 122 ++++++++++++++++++++++++ hack/openshift/run-e2e-tests.sh | 77 +++++++++++++++ 5 files changed, 359 insertions(+) create mode 100644 hack/openshift/manifests/cma.yaml create mode 100644 hack/openshift/manifests/gitea.yaml create mode 100644 hack/openshift/manifests/pipelines.yaml create mode 100755 hack/openshift/prepare-ocp-cluster.sh create mode 100755 hack/openshift/run-e2e-tests.sh diff --git a/hack/openshift/manifests/cma.yaml b/hack/openshift/manifests/cma.yaml new file mode 100644 index 00000000..cb03acbd --- /dev/null +++ b/hack/openshift/manifests/cma.yaml @@ -0,0 +1,27 @@ +--- +# Custom Metrics Autoscaler (productized KEDA), including the HTTP add-on. +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-keda +--- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: openshift-keda + namespace: openshift-keda +spec: + targetNamespaces: + - openshift-keda +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: openshift-custom-metrics-autoscaler-operator + namespace: openshift-keda +spec: + channel: stable + name: openshift-custom-metrics-autoscaler-operator + source: redhat-operators + sourceNamespace: openshift-marketplace + installPlanApproval: Automatic diff --git a/hack/openshift/manifests/gitea.yaml b/hack/openshift/manifests/gitea.yaml new file mode 100644 index 00000000..de6c36e0 --- /dev/null +++ b/hack/openshift/manifests/gitea.yaml @@ -0,0 +1,118 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: gitea +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gitea + namespace: gitea + labels: + app: gitea +spec: + replicas: 1 + selector: + matchLabels: + app: gitea + template: + metadata: + labels: + app: gitea + spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: gitea + # rootless image: runs as an arbitrary non-root UID (required by OCP restricted SCC) + image: docker.io/gitea/gitea:1.22-rootless + imagePullPolicy: IfNotPresent + env: + - name: GITEA__security__INSTALL_LOCK + value: "true" + - name: GITEA__database__DB_TYPE + value: sqlite3 + - name: GITEA__database__PATH + value: /var/lib/gitea/data/gitea.db + - name: GITEA__server__HTTP_PORT + value: "3000" + - name: GITEA__server__SSH_PORT + value: "2222" + ports: + - name: http + containerPort: 3000 + - name: ssh + containerPort: 2222 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readinessProbe: + httpGet: + path: /api/healthz + port: 3000 + initialDelaySeconds: 10 + periodSeconds: 5 + # Create the admin user the test suite authenticates as (giteaadmin/giteapass). + # Idempotent: retries until Gitea is up, ignores "already exists". + lifecycle: + postStart: + exec: + command: + - /bin/sh + - -c + - | + for i in $(seq 1 30); do + if gitea admin user create --admin \ + --username giteaadmin --password giteapass \ + --email admin@gitea.local --must-change-password=false; then + exit 0 + fi + sleep 5 + done + volumeMounts: + - name: data + mountPath: /var/lib/gitea + - name: config + mountPath: /etc/gitea + volumes: + - name: data + emptyDir: {} + - name: config + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: gitea + namespace: gitea + labels: + app: gitea +spec: + selector: + app: gitea + ports: + - name: http + port: 3000 + targetPort: 3000 + - name: ssh + port: 2222 + targetPort: 2222 +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: gitea + namespace: gitea + labels: + app: gitea +spec: + to: + kind: Service + name: gitea + port: + targetPort: http diff --git a/hack/openshift/manifests/pipelines.yaml b/hack/openshift/manifests/pipelines.yaml new file mode 100644 index 00000000..95c80f83 --- /dev/null +++ b/hack/openshift/manifests/pipelines.yaml @@ -0,0 +1,15 @@ +--- +# Red Hat OpenShift Pipelines (productized Tekton). +# Installed into openshift-operators, which already has a global (AllNamespaces) +# OperatorGroup, so no OperatorGroup is needed here. +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: openshift-pipelines-operator-rh + namespace: openshift-operators +spec: + channel: latest + name: openshift-pipelines-operator-rh + source: redhat-operators + sourceNamespace: openshift-marketplace + installPlanApproval: Automatic diff --git a/hack/openshift/prepare-ocp-cluster.sh b/hack/openshift/prepare-ocp-cluster.sh new file mode 100755 index 00000000..ea70d5c4 --- /dev/null +++ b/hack/openshift/prepare-ocp-cluster.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# +# Prepare an existing OpenShift cluster for the functions-operator e2e suite. +# Unlike hack/create-kind-cluster.sh this does NOT create a cluster: it installs +# only the dependencies the suite needs onto the cluster your current `oc` context +# points at. +# +# Installs the full dependency set by default (each gated by an env flag): +# - Gitea + the kube-public/gitea-endpoint ConfigMap the suite reads +# - Red Hat OpenShift Pipelines (productized Tekton) for on-cluster func builds +# - Custom Metrics Autoscaler (productized KEDA) incl. the HTTP add-on, for the +# keda deployer +# Opt out of any piece with INSTALL_GITEA=false / INSTALL_PIPELINES=false / +# INSTALL_KEDA=false. Preparing everything means one prep covers both raw and keda. +# +# No container registry is deployed: the suite uses ttl.sh (set REGISTRY=ttl.sh/...) +# together with on-cluster builds (FUNC_REMOTE=true), which produce a ttl.sh-servable +# image. See the e2e run recipe in the repo docs. + +set -o errexit +set -o nounset +set -o pipefail + +BASEDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MANIFESTS="${BASEDIR}/manifests" + +INSTALL_GITEA="${INSTALL_GITEA:-true}" +INSTALL_PIPELINES="${INSTALL_PIPELINES:-true}" +# Prepare the cluster fully by default so it's ready for both the raw and keda +# deployers without re-running prep. CMA (KEDA) is a handful of extra pods and does +# not interfere with raw runs. Opt out with INSTALL_KEDA=false. +INSTALL_KEDA="${INSTALL_KEDA:-true}" + +header=$'\e[1;33m' +reset=$'\e[0m' +function header_text { echo "$header$*$reset"; } + +# Wait for an OLM subscription's CSV to reach Succeeded. +# args: +function wait_for_csv() { + local sub="$1" ns="$2" csv="" + header_text "Waiting for CSV of subscription ${sub} in ${ns}" + for _ in $(seq 1 60); do + csv="$(oc get subscription "$sub" -n "$ns" -o jsonpath='{.status.installedCSV}' 2>/dev/null || true)" + if [ -n "$csv" ] && [ "$(oc get csv "$csv" -n "$ns" -o jsonpath='{.status.phase}' 2>/dev/null || true)" = "Succeeded" ]; then + header_text "CSV ${csv} ready" + return 0 + fi + sleep 10 + done + echo "Timed out waiting for CSV of ${sub} in ${ns}" >&2 + return 1 +} + +function install_gitea() { + header_text "Installing Gitea" + oc apply -f "${MANIFESTS}/gitea.yaml" + oc rollout status deploy/gitea -n gitea --timeout=180s + + local host + host="$(oc get route gitea -n gitea -o jsonpath='{.spec.host}')" + + # The suite (test/utils/gitea.go) discovers Gitea via this ConfigMap. It requires + # both keys. The SSH endpoint must be reachable from inside the cluster (the + # operator's syncer and the Tekton git-clone task clone over it; the test host + # only builds the URL string, it never dials SSH). The Route only serves HTTP, + # so SSH points at the in-cluster service, not the route host. + header_text "Writing kube-public/gitea-endpoint ConfigMap (http://${host})" + oc create configmap gitea-endpoint -n kube-public \ + --from-literal="http=http://${host}" \ + --from-literal="ssh=gitea.gitea.svc:2222" \ + --dry-run=client -o yaml | oc apply -f - +} + +function install_pipelines() { + header_text "Installing Red Hat OpenShift Pipelines" + oc apply -f "${MANIFESTS}/pipelines.yaml" + wait_for_csv openshift-pipelines-operator-rh openshift-operators + + header_text "Waiting for TektonConfig to be created" + for _ in $(seq 1 30); do + oc get tektonconfig config >/dev/null 2>&1 && break + sleep 5 + done + + header_text "Waiting for TektonPipeline to be ready" + oc wait --for=condition=Ready --timeout=300s tektonpipeline/pipeline || true + + # Disable Tekton Results. Productized OpenShift Pipelines enables it by default; its + # results.tekton.dev/taskrun finalizer races with test-namespace deletion and hangs + # cleanup when the namespace is deleted before the Results watcher clears the finalizer. + # We don't need run history for e2e, so turn it off for deterministic teardown. + # (Field is singular "result"; plural "results" is silently ignored.) + header_text "Disabling Tekton Results" + oc patch tektonconfig config --type=merge -p '{"spec":{"result":{"disabled":true}}}' +} + +function install_keda() { + header_text "Installing Custom Metrics Autoscaler (productized KEDA)" + oc apply -f "${MANIFESTS}/cma.yaml" + wait_for_csv openshift-custom-metrics-autoscaler-operator openshift-keda + + # The operator auto-creates a KedaController named "keda"; wait for it, then enable + # the HTTP add-on, which func's keda deployer needs (HTTPScaledObject). Off by default. + header_text "Enabling the KEDA HTTP add-on" + for _ in $(seq 1 30); do + oc get kedacontroller keda -n openshift-keda >/dev/null 2>&1 && break + sleep 5 + done + oc patch kedacontroller keda -n openshift-keda --type=merge \ + -p '{"spec":{"watchNamespace":"","httpAddon":{"enabled":true}}}' + + header_text "Waiting for KEDA HTTP add-on pods" + oc rollout status deploy/keda-add-ons-http-interceptor -n openshift-keda --timeout=180s || true + oc rollout status deploy/keda-add-ons-http-scaler -n openshift-keda --timeout=180s || true +} + +[ "$INSTALL_GITEA" = "true" ] && install_gitea +[ "$INSTALL_PIPELINES" = "true" ] && install_pipelines +[ "$INSTALL_KEDA" = "true" ] && install_keda + +header_text "Cluster prepared for functions-operator e2e" diff --git a/hack/openshift/run-e2e-tests.sh b/hack/openshift/run-e2e-tests.sh new file mode 100755 index 00000000..88f992ea --- /dev/null +++ b/hack/openshift/run-e2e-tests.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# +# Run the functions-operator e2e suite against the current OpenShift cluster +# using ttl.sh as the image registry, with all builds and pushes on-cluster. +# +# Prerequisites (see prepare-ocp-cluster.sh in this directory): +# - operator deployed into func-operator-system (make deploy KUBECTL=oc IMG=...) +# - Gitea + the kube-public/gitea-endpoint ConfigMap +# - Red Hat OpenShift Pipelines (the on-cluster builder) +# - Custom Metrics Autoscaler when DEFAULT_DEPLOYER=keda +# +# Why ttl.sh works only with FUNC_REMOTE=true: the local func pusher wraps +# every pushed image in a single-entry manifest list and re-points the tag at +# it; ttl.sh then fails to serve the per-arch child manifest ("manifest +# unknown"). On-cluster (Tekton/buildah) pushes are plain single manifests, +# which ttl.sh serves fine. +# +# Why FUNC_MANAGEMENT_DISABLED=true: every e2e spec creates and asserts on its +# own Function CR. Without this, each seed `func deploy` also syncs a second CR +# that the operator reconciles out of the spec's control, racing it. +# +# Both variables are read natively by the func CLI (viper FUNC_ env +# binding) and inherited by every func invocation the suite makes. +# +# Environment (all optional, defaults shown): +# DEFAULT_DEPLOYER raw | keda (keda needs the CMA installed) +# E2E_SKIP regex of spec texts to skip. The default drops the specs +# that cannot work under FUNC_REMOTE=true: +# SSH - the SSH-repository specs need an SSH endpoint +# route the cluster may not expose +# Middleware Update, registry auth secret - both seed +# with the old func CLI and rely on that seed +# carrying stale middleware. Remotely, the old +# CLI's pipeline scaffolds with whatever the +# mutable upstream ghcr.io/knative/func-utils:v2 +# tag currently serves, so the seed is never +# stale and the specs' premise breaks. +# monorepo - the seed's remote deploy does not leave +# the function name in the committed func.yaml +# the way a local deploy does, so the operator's +# metadata read comes back with an empty name and +# every reconcile dies at `func describe -n ` +# with "must also specify a name when specifying +# namespace" (Deployed/Ready never become True). +# STALE_FUNCTION_IMAGE pre-built stale-middleware image for the +# middleware-rebuild spec; default unset => spec skips +# itself. NOTE: with FUNC_REMOTE=true the spec's digested +# seed deploy cannot go remote (func only skips build/push +# for digested refs on the local path), so the spec fails +# in its seed until the utils gate that; leave unset for now. +# FUNC_BIN func binary to use (default "func" on PATH) +# REGISTRY push registry prefix (default: random ttl.sh namespace) + +set -o errexit +set -o nounset +set -o pipefail + +# A fresh random ttl.sh namespace per run so pushes never collide; ttl.sh +# expires images within 24h by tag. +REGISTRY="${REGISTRY:-ttl.sh/funcop-e2e-$(head -c 128