diff --git a/ci-operator/config/openshift/hypershift/openshift-hypershift-main.yaml b/ci-operator/config/openshift/hypershift/openshift-hypershift-main.yaml index 5e904576c36e..55b06f135760 100644 --- a/ci-operator/config/openshift/hypershift/openshift-hypershift-main.yaml +++ b/ci-operator/config/openshift/hypershift/openshift-hypershift-main.yaml @@ -296,6 +296,47 @@ tests: TECH_PREVIEW_NO_UPGRADE: "true" TEST_SUITE: openshift/conformance/parallel/minimal workflow: hypershift-aws-e2e-backuprestore +- always_run: false + as: e2e-v2-aws-etcd-sharding + capabilities: + - arm64 + optional: true + steps: + cluster_profile: hypershift-aws + env: + NODEPOOL_N1_VERSION: "5.0" + TECH_PREVIEW_NO_UPGRADE: "true" + TEST_PLAN_FILE: /hypershift/testplans/aws-etcd-sharding.yaml + workflow: hypershift-aws-e2e-v2 +- always_run: false + as: e2e-v2-azure-self-managed-etcd-sharding + capabilities: + - arm64 + optional: true + steps: + cluster_profile: hypershift-azure + env: + ENABLE_HYPERSHIFT_CERT_ROTATION_SCALE: "true" + HYPERSHIFT_AZURE_LOCATION: centralus + NODEPOOL_N1_VERSION: "5.0" + TECH_PREVIEW_NO_UPGRADE: "true" + TEST_PLAN_FILE: /hypershift/testplans/azure-etcd-sharding.yaml + workflow: hypershift-azure-e2e-v2-self-managed +- always_run: false + as: e2e-v2-aws-backuprestore-etcd-sharding + capabilities: + - arm64 + optional: true + steps: + cluster_profile: hypershift-aws + env: + HYPERSHIFT_CP_AVAILABILITY_POLICY: HighlyAvailable + HYPERSHIFT_ETCD_SHARDS: |- + name=events,resources=/events;events.k8s.io/events,replicas=3,storage=EmptyDir + name=leases,resources=coordination.k8s.io/leases,replicas=3,storage=PersistentVolume + TECH_PREVIEW_NO_UPGRADE: "true" + TEST_SUITE: openshift/conformance/parallel/minimal + workflow: hypershift-aws-e2e-backuprestore - always_run: false as: e2e-kubevirt-metal-ovn-backuprestore capabilities: @@ -795,6 +836,30 @@ tests: helmValues: controlPlaneOperatorImage: IMAGE_REPO:IMAGE_TAG workflow: rosa-seaplane-ephemeral-e2e +- as: e2e-aws-etcd-sharding + capabilities: + - arm64 + cron: 15 3 * * * + steps: + cluster_profile: hypershift-aws + env: + NODEPOOL_N1_VERSION: "5.0" + TECH_PREVIEW_NO_UPGRADE: "true" + TEST_PLAN_FILE: /hypershift/testplans/aws-etcd-sharding.yaml + workflow: hypershift-aws-e2e-v2 +- as: e2e-azure-etcd-sharding + capabilities: + - arm64 + cron: 15 5 * * 3 + steps: + cluster_profile: hypershift-azure + env: + ENABLE_HYPERSHIFT_CERT_ROTATION_SCALE: "true" + HYPERSHIFT_AZURE_LOCATION: centralus + NODEPOOL_N1_VERSION: "5.0" + TECH_PREVIEW_NO_UPGRADE: "true" + TEST_PLAN_FILE: /hypershift/testplans/azure-etcd-sharding.yaml + workflow: hypershift-azure-e2e-v2-self-managed zz_generated_metadata: branch: main org: openshift diff --git a/ci-operator/config/openshift/hypershift/openshift-hypershift-main__periodics-etcd-perf.yaml b/ci-operator/config/openshift/hypershift/openshift-hypershift-main__periodics-etcd-perf.yaml new file mode 100644 index 000000000000..59298f238957 --- /dev/null +++ b/ci-operator/config/openshift/hypershift/openshift-hypershift-main__periodics-etcd-perf.yaml @@ -0,0 +1,71 @@ +base_images: + hypershift-operator: + name: hypershift-operator + namespace: hypershift + tag: latest + hypershift-tests: + name: hypershift-tests + namespace: hypershift + tag: latest + upi-installer: + name: "5.1" + namespace: ocp + tag: upi-installer +prowgen: + enable_secrets_store_csi_driver: true +releases: + initial: + candidate: + product: ocp + stream: ci + version: "5.1" + latest: + candidate: + product: ocp + stream: ci + version: "5.1" +resources: + '*': + requests: + cpu: 100m + memory: 200Mi +tests: +- as: annotation-churn-standard + cron: 20 7 * * 2 + steps: + cluster_profile: hypershift-aws + env: + ANNOTATION_CHURN_ITERATIONS: "10" + HYPERSHIFT_CP_AVAILABILITY_POLICY: HighlyAvailable + HYPERSHIFT_INSTANCE_TYPE: m5.xlarge + HYPERSHIFT_NODE_COUNT: "24" + PATCH_REPLICAS: "50" + PATCH_ROUNDS: "100" + test: + - chain: hypershift-etcd-perf + workflow: hypershift-aws-e2e-cluster + timeout: 6h0m0s +- as: annotation-churn-sharded + cron: 20 13 * * 2 + steps: + cluster_profile: hypershift-aws + env: + ANNOTATION_CHURN_ITERATIONS: "10" + HYPERSHIFT_CP_AVAILABILITY_POLICY: HighlyAvailable + HYPERSHIFT_ETCD_SHARDS: |- + name=leases,resources=coordination.k8s.io/leases,replicas=3,storage=PersistentVolume + name=events,resources=/events;events.k8s.io/events,replicas=1,storage=EmptyDir + HYPERSHIFT_INSTANCE_TYPE: m5.xlarge + HYPERSHIFT_NODE_COUNT: "24" + PATCH_REPLICAS: "50" + PATCH_ROUNDS: "100" + TECH_PREVIEW_NO_UPGRADE: "true" + test: + - chain: hypershift-etcd-perf + workflow: hypershift-aws-e2e-cluster + timeout: 6h0m0s +zz_generated_metadata: + branch: main + org: openshift + repo: hypershift + variant: periodics-etcd-perf diff --git a/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-periodics.yaml b/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-periodics.yaml index eb07e50180f9..61111be1b902 100644 --- a/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-periodics.yaml +++ b/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-periodics.yaml @@ -298,6 +298,224 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build07 + cron: 15 3 * * * + decorate: true + decoration_config: + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile + - Dockerfile.control-plane + - Dockerfile.e2e + extra_refs: + - base_ref: main + org: openshift + repo: hypershift + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile + - Dockerfile.control-plane + - Dockerfile.e2e + labels: + capability/arm64: arm64 + ci-operator.openshift.io/cloud: hypershift-aws + ci-operator.openshift.io/cloud-cluster-profile: hypershift-aws + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-hypershift-main-e2e-aws-etcd-sharding + spec: + containers: + - args: + - --enable-secrets-store-csi-driver=true + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --gsm-config=/etc/gsm-config/gsm-config.yaml + - --gsm-credentials-file=/etc/gsm-credentials/key.json + - --gsm-project-config=/etc/gsm-config/gsm-project-config.yaml + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=e2e-aws-etcd-sharding + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/gsm-config + name: gsm-config + readOnly: true + - mountPath: /etc/gsm-credentials + name: gsm-sa-key + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - configMap: + name: gsm-config + name: gsm-config + - csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: ci-operator-sa-key-spc + name: gsm-sa-key + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator +- agent: kubernetes + cluster: build07 + cron: 15 5 * * 3 + decorate: true + decoration_config: + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile + - Dockerfile.control-plane + - Dockerfile.e2e + extra_refs: + - base_ref: main + org: openshift + repo: hypershift + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile + - Dockerfile.control-plane + - Dockerfile.e2e + labels: + capability/arm64: arm64 + ci-operator.openshift.io/cloud: hypershift-azure + ci-operator.openshift.io/cloud-cluster-profile: hypershift-azure + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-hypershift-main-e2e-azure-etcd-sharding + spec: + containers: + - args: + - --enable-secrets-store-csi-driver=true + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --gsm-config=/etc/gsm-config/gsm-config.yaml + - --gsm-credentials-file=/etc/gsm-credentials/key.json + - --gsm-project-config=/etc/gsm-config/gsm-project-config.yaml + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=e2e-azure-etcd-sharding + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/gsm-config + name: gsm-config + readOnly: true + - mountPath: /etc/gsm-credentials + name: gsm-sa-key + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - configMap: + name: gsm-config + name: gsm-config + - csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: ci-operator-sa-key-spc + name: gsm-sa-key + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator - agent: kubernetes cluster: build07 cron: '*/30 * * * *' @@ -602,6 +820,212 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build05 + cron: 20 13 * * 2 + decorate: true + decoration_config: + skip_cloning: true + timeout: 6h0m0s + extra_refs: + - base_ref: main + org: openshift + repo: hypershift + labels: + ci-operator.openshift.io/cloud: hypershift-aws + ci-operator.openshift.io/cloud-cluster-profile: hypershift-aws + ci-operator.openshift.io/variant: periodics-etcd-perf + ci.openshift.io/generator: prowgen + job-release: "5.1" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-hypershift-main-periodics-etcd-perf-annotation-churn-sharded + spec: + containers: + - args: + - --enable-secrets-store-csi-driver=true + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --gsm-config=/etc/gsm-config/gsm-config.yaml + - --gsm-credentials-file=/etc/gsm-credentials/key.json + - --gsm-project-config=/etc/gsm-config/gsm-project-config.yaml + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=annotation-churn-sharded + - --variant=periodics-etcd-perf + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/gsm-config + name: gsm-config + readOnly: true + - mountPath: /etc/gsm-credentials + name: gsm-sa-key + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - configMap: + name: gsm-config + name: gsm-config + - csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: ci-operator-sa-key-spc + name: gsm-sa-key + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator +- agent: kubernetes + cluster: build05 + cron: 20 7 * * 2 + decorate: true + decoration_config: + skip_cloning: true + timeout: 6h0m0s + extra_refs: + - base_ref: main + org: openshift + repo: hypershift + labels: + ci-operator.openshift.io/cloud: hypershift-aws + ci-operator.openshift.io/cloud-cluster-profile: hypershift-aws + ci-operator.openshift.io/variant: periodics-etcd-perf + ci.openshift.io/generator: prowgen + job-release: "5.1" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-hypershift-main-periodics-etcd-perf-annotation-churn-standard + spec: + containers: + - args: + - --enable-secrets-store-csi-driver=true + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --gsm-config=/etc/gsm-config/gsm-config.yaml + - --gsm-credentials-file=/etc/gsm-credentials/key.json + - --gsm-project-config=/etc/gsm-config/gsm-project-config.yaml + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=annotation-churn-standard + - --variant=periodics-etcd-perf + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/gsm-config + name: gsm-config + readOnly: true + - mountPath: /etc/gsm-credentials + name: gsm-sa-key + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - configMap: + name: gsm-config + name: gsm-config + - csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: ci-operator-sa-key-spc + name: gsm-sa-key + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator - agent: kubernetes cluster: build05 cron: 0 8 * * * diff --git a/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-presubmits.yaml b/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-presubmits.yaml index aa921b6dc52a..880a3bae2074 100644 --- a/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-presubmits.yaml +++ b/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-presubmits.yaml @@ -4075,6 +4075,220 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )e2e-v2-aws-backuprestore,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build09 + context: ci/prow/e2e-v2-aws-backuprestore-etcd-sharding + decorate: true + decoration_config: + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile + - Dockerfile.control-plane + - Dockerfile.e2e + labels: + capability/arm64: arm64 + ci-operator.openshift.io/cloud: hypershift-aws + ci-operator.openshift.io/cloud-cluster-profile: hypershift-aws + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-hypershift-main-e2e-v2-aws-backuprestore-etcd-sharding + optional: true + rerun_command: /test e2e-v2-aws-backuprestore-etcd-sharding + spec: + containers: + - args: + - --enable-secrets-store-csi-driver=true + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --gsm-config=/etc/gsm-config/gsm-config.yaml + - --gsm-credentials-file=/etc/gsm-credentials/key.json + - --gsm-project-config=/etc/gsm-config/gsm-project-config.yaml + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=e2e-v2-aws-backuprestore-etcd-sharding + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/gsm-config + name: gsm-config + readOnly: true + - mountPath: /etc/gsm-credentials + name: gsm-sa-key + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - configMap: + name: gsm-config + name: gsm-config + - csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: ci-operator-sa-key-spc + name: gsm-sa-key + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )e2e-v2-aws-backuprestore-etcd-sharding,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build09 + context: ci/prow/e2e-v2-aws-etcd-sharding + decorate: true + decoration_config: + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile + - Dockerfile.control-plane + - Dockerfile.e2e + labels: + capability/arm64: arm64 + ci-operator.openshift.io/cloud: hypershift-aws + ci-operator.openshift.io/cloud-cluster-profile: hypershift-aws + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-hypershift-main-e2e-v2-aws-etcd-sharding + optional: true + rerun_command: /test e2e-v2-aws-etcd-sharding + spec: + containers: + - args: + - --enable-secrets-store-csi-driver=true + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --gsm-config=/etc/gsm-config/gsm-config.yaml + - --gsm-credentials-file=/etc/gsm-credentials/key.json + - --gsm-project-config=/etc/gsm-config/gsm-project-config.yaml + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=e2e-v2-aws-etcd-sharding + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/gsm-config + name: gsm-config + readOnly: true + - mountPath: /etc/gsm-credentials + name: gsm-sa-key + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - configMap: + name: gsm-config + name: gsm-config + - csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: ci-operator-sa-key-spc + name: gsm-sa-key + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )e2e-v2-aws-etcd-sharding,?($|\s.*) - agent: kubernetes always_run: false annotations: @@ -4183,6 +4397,113 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )e2e-v2-azure-self-managed,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build09 + context: ci/prow/e2e-v2-azure-self-managed-etcd-sharding + decorate: true + decoration_config: + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile + - Dockerfile.control-plane + - Dockerfile.e2e + labels: + capability/arm64: arm64 + ci-operator.openshift.io/cloud: hypershift-azure + ci-operator.openshift.io/cloud-cluster-profile: hypershift-azure + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-hypershift-main-e2e-v2-azure-self-managed-etcd-sharding + optional: true + rerun_command: /test e2e-v2-azure-self-managed-etcd-sharding + spec: + containers: + - args: + - --enable-secrets-store-csi-driver=true + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --gsm-config=/etc/gsm-config/gsm-config.yaml + - --gsm-credentials-file=/etc/gsm-credentials/key.json + - --gsm-project-config=/etc/gsm-config/gsm-project-config.yaml + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=e2e-v2-azure-self-managed-etcd-sharding + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/gsm-config + name: gsm-config + readOnly: true + - mountPath: /etc/gsm-credentials + name: gsm-sa-key + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - configMap: + name: gsm-config + name: gsm-config + - csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: ci-operator-sa-key-spc + name: gsm-sa-key + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )e2e-v2-azure-self-managed-etcd-sharding,?($|\s.*) - agent: kubernetes always_run: false annotations: diff --git a/ci-operator/step-registry/hypershift/aws/create-guests/hypershift-aws-create-guests-commands.sh b/ci-operator/step-registry/hypershift/aws/create-guests/hypershift-aws-create-guests-commands.sh index 52ecf9d24c17..ce160e4c7c65 100644 --- a/ci-operator/step-registry/hypershift/aws/create-guests/hypershift-aws-create-guests-commands.sh +++ b/ci-operator/step-registry/hypershift/aws/create-guests/hypershift-aws-create-guests-commands.sh @@ -11,7 +11,11 @@ if [[ -f "${SHARED_DIR}/nodepool_release_images" ]]; then fi if [[ -f "${SHARED_DIR}/test-plan.yaml" ]]; then + # An inline plan written by hypershift-write-test-plan wins over a plan + # that ships in the image, so a job can always override. export TEST_PLAN="${SHARED_DIR}/test-plan.yaml" +elif [[ -n "${TEST_PLAN_FILE:-}" ]]; then + export TEST_PLAN="${TEST_PLAN_FILE}" fi # Storage KMS encryption: only forward the KMS key alias when the hypershift diff --git a/ci-operator/step-registry/hypershift/aws/create-guests/hypershift-aws-create-guests-ref.yaml b/ci-operator/step-registry/hypershift/aws/create-guests/hypershift-aws-create-guests-ref.yaml index 931a6e56f203..2d4b66fe776e 100644 --- a/ci-operator/step-registry/hypershift/aws/create-guests/hypershift-aws-create-guests-ref.yaml +++ b/ci-operator/step-registry/hypershift/aws/create-guests/hypershift-aws-create-guests-ref.yaml @@ -39,6 +39,9 @@ ref: - name: HYPERSHIFT_STORAGE_KMS_KEY_ALIAS default: "" documentation: "AWS KMS key alias (e.g. alias/hypershift-ci) used to encrypt the default StorageClass volumes on the public cluster. When set and supported by the hypershift binary, the alias is resolved to an ARN and passed as --initial-storage-volumes-kms-key at cluster creation. Leave empty to skip." + - name: TEST_PLAN_FILE + default: "" + documentation: "Path to a test plan that ships in the hypershift-tests image, e.g. /hypershift/testplans/aws-etcd-sharding.yaml. Ignored when hypershift-write-test-plan has written an inline plan to SHARED_DIR. If both are empty, the platform's default test plan is used." from: hypershift-tests grace_period: 5m0s resources: diff --git a/ci-operator/step-registry/hypershift/aws/create/hypershift-aws-create-chain.yaml b/ci-operator/step-registry/hypershift/aws/create/hypershift-aws-create-chain.yaml index aa3dbdff06de..26351ba84107 100644 --- a/ci-operator/step-registry/hypershift/aws/create/hypershift-aws-create-chain.yaml +++ b/ci-operator/step-registry/hypershift/aws/create/hypershift-aws-create-chain.yaml @@ -33,6 +33,21 @@ chain: - name: EXTRA_ARGS default: "" documentation: "Extra args to pass to the create cluster aws command" + - name: HYPERSHIFT_ETCD_SHARDS + default: "" + documentation: | + Newline separated --etcd-shard values, one per managed etcd shard. Each value has the form + "name=,resources=/[;/...][,replicas=1|3][,storage=PersistentVolume|EmptyDir][,storageClassName=]"; + use an empty apiGroup for the core group. Requires the EtcdSharding feature gate, so the + HyperShift operator must be installed with TECH_PREVIEW_NO_UPGRADE=true, and shards with + replicas=3 require HYPERSHIFT_CP_AVAILABILITY_POLICY=HighlyAvailable. + + The topology the performance jobs use, agreed with the performance and scale team, is a + three member PersistentVolume backed main etcd plus: + name=leases,resources=coordination.k8s.io/leases,replicas=3,storage=PersistentVolume + name=events,resources=/events;events.k8s.io/events,replicas=1,storage=EmptyDir + Leases get their own highly available shard because they carry the cluster heartbeat, and + events get a single EmptyDir shard because they are high churn and disposable. - name: HYPERSHIFT_GUEST_INFRA_OCP_ACCOUNT default: "false" documentation: "Whether to use the generic CI account or the HyperShift OSD account for the guest clusters infra. For the infra created for the clusters. E.g. For cluster-bot we use the generic CI account" @@ -105,13 +120,15 @@ chain: AWS_GUEST_INFRA_CREDENTIALS_FILE="/etc/hypershift-ci-jobs-awscreds/credentials" DEFAULT_BASE_DOMAIN=ci.hypershift.devcluster.openshift.com + # Renders the HostedCluster, rewrites the NodePool release image and applies + # the result. COMMAND is passed as an argument array rather than a shell + # string so that values containing shell metacharacters -- most notably the + # ";" that separates resources inside an --etcd-shard value -- survive. support_np_skew() { curl -L https://github.com/mikefarah/yq/releases/download/v4.31.2/yq_linux_amd64 -o /tmp/yq && chmod +x /tmp/yq - local extra_flags="" - extra_flags+="--render-sensitive --render > /tmp/hc.yaml " - extra_flags+="&& /tmp/yq e -i '(select(.kind == \"NodePool\").spec.release.image) = \"$NODEPOOL_RELEASE_IMAGE_LATEST\"' /tmp/hc.yaml " - extra_flags+="&& oc apply -f /tmp/hc.yaml" - echo "$extra_flags" + "${COMMAND[@]}" --render-sensitive --render > /tmp/hc.yaml + /tmp/yq e -i "(select(.kind == \"NodePool\").spec.release.image) = \"$NODEPOOL_RELEASE_IMAGE_LATEST\"" /tmp/hc.yaml + oc apply -f /tmp/hc.yaml } HC_REGION=${HYPERSHIFT_AWS_REGION:-$LEASED_RESOURCE} @@ -174,6 +191,15 @@ chain: COMMAND+=(--network-type="${HYPERSHIFT_NETWORK_TYPE}") fi + # Shard values contain ";" as the resource separator, so they get their + # own quoted knob rather than riding along in the word-split EXTRA_ARGS. + if [[ -n $HYPERSHIFT_ETCD_SHARDS ]]; then + while IFS= read -r shard; do + [[ -n "$shard" ]] || continue + COMMAND+=(--etcd-shard="$shard") + done <<< "$HYPERSHIFT_ETCD_SHARDS" + fi + if [[ -n $EXTRA_ARGS ]]; then COMMAND+=(${EXTRA_ARGS}) fi @@ -230,9 +256,8 @@ chain: fi # Execute the command - # When using np_skew (different release images), we need eval to properly handle shell operators if [[ -n "$RELEASE_IMAGE_LATEST" && -n "$NODEPOOL_RELEASE_IMAGE_LATEST" && "$RELEASE_IMAGE_LATEST" != "$NODEPOOL_RELEASE_IMAGE_LATEST" ]]; then - eval "${COMMAND[*]} $(support_np_skew)" + support_np_skew else "${COMMAND[@]}" fi diff --git a/ci-operator/step-registry/hypershift/aws/run-e2e-v2/hypershift-aws-run-e2e-v2-chain.yaml b/ci-operator/step-registry/hypershift/aws/run-e2e-v2/hypershift-aws-run-e2e-v2-chain.yaml index fb5f9b57520c..b8d11ab3f0df 100644 --- a/ci-operator/step-registry/hypershift/aws/run-e2e-v2/hypershift-aws-run-e2e-v2-chain.yaml +++ b/ci-operator/step-registry/hypershift/aws/run-e2e-v2/hypershift-aws-run-e2e-v2-chain.yaml @@ -7,6 +7,9 @@ chain: - name: HYPERSHIFT_PLATFORM default: "" documentation: "The cloud platform for the clusters (e.g. aws). Set in the workflow env block." + - name: TEST_PLAN_FILE + default: "" + documentation: "Path to a test plan that ships in the hypershift-tests image, e.g. /hypershift/testplans/aws-etcd-sharding.yaml. Must match the value given to hypershift-aws-create-guests so the same plan selects the clusters and the tests. Ignored when hypershift-write-test-plan has written an inline plan to SHARED_DIR." commands: |- set -xuo pipefail export KUBECONFIG="${SHARED_DIR}/management_cluster_kubeconfig" @@ -14,7 +17,11 @@ chain: export AWS_GUEST_INFRA_CREDENTIALS_FILE="/etc/hypershift-ci-jobs-awscreds/credentials" export PULL_SECRET_FILE="/etc/ci-pull-credentials/.dockerconfigjson" if [[ -f "${SHARED_DIR}/test-plan.yaml" ]]; then + # An inline plan written by hypershift-write-test-plan wins over a + # plan that ships in the image, so a job can always override. export TEST_PLAN="${SHARED_DIR}/test-plan.yaml" + elif [[ -n "${TEST_PLAN_FILE:-}" ]]; then + export TEST_PLAN="${TEST_PLAN_FILE}" fi /hypershift/bin/run-tests timeout: 150m diff --git a/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh b/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh index 0341fb58a9d5..047b635a4f7d 100644 --- a/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh +++ b/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh @@ -10,4 +10,8 @@ if [[ -f "${SHARED_DIR}/nodepool_release_images" ]]; then source "${SHARED_DIR}/nodepool_release_images" fi +if [[ -n "${TEST_PLAN_FILE:-}" ]]; then + export TEST_PLAN="${TEST_PLAN_FILE}" +fi + /hypershift/bin/create-guests diff --git a/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-ref.yaml b/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-ref.yaml index 6fa8ef5a8b3d..3f6e8cb55566 100644 --- a/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-ref.yaml +++ b/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-ref.yaml @@ -48,6 +48,9 @@ ref: - name: HYPERSHIFT_PLATFORM default: "" documentation: "The cloud platform for the self-managed clusters (e.g. azure). Set in the workflow env block." + - name: TEST_PLAN_FILE + default: "" + documentation: "Path to a test plan that ships in the hypershift-tests image, e.g. /hypershift/testplans/azure-etcd-sharding.yaml. If empty, the platform's default test plan is used." from: hypershift-tests grace_period: 5m0s resources: diff --git a/ci-operator/step-registry/hypershift/azure/run-e2e-v2-selfmanaged/hypershift-azure-run-e2e-v2-selfmanaged-chain.yaml b/ci-operator/step-registry/hypershift/azure/run-e2e-v2-selfmanaged/hypershift-azure-run-e2e-v2-selfmanaged-chain.yaml index c10066ef0dc3..2837395a7738 100644 --- a/ci-operator/step-registry/hypershift/azure/run-e2e-v2-selfmanaged/hypershift-azure-run-e2e-v2-selfmanaged-chain.yaml +++ b/ci-operator/step-registry/hypershift/azure/run-e2e-v2-selfmanaged/hypershift-azure-run-e2e-v2-selfmanaged-chain.yaml @@ -14,9 +14,15 @@ chain: - name: HYPERSHIFT_PLATFORM default: "" documentation: "The cloud platform for the self-managed clusters (e.g. azure). Set in the workflow env block." + - name: TEST_PLAN_FILE + default: "" + documentation: "Path to a test plan that ships in the hypershift-tests image, e.g. /hypershift/testplans/azure-etcd-sharding.yaml. Must match the value given to hypershift-azure-create-selfmanaged-guests so the same plan selects the clusters and the tests." commands: |- set -xuo pipefail export KUBECONFIG="${SHARED_DIR}/management_cluster_kubeconfig" + if [[ -n "${TEST_PLAN_FILE:-}" ]]; then + export TEST_PLAN="${TEST_PLAN_FILE}" + fi /hypershift/bin/run-tests timeout: 150m grace_period: 5m diff --git a/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/OWNERS b/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/OWNERS new file mode 120000 index 000000000000..ec405d65a79d --- /dev/null +++ b/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/OWNERS @@ -0,0 +1 @@ +../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/hypershift-enable-user-workload-monitoring-commands.sh b/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/hypershift-enable-user-workload-monitoring-commands.sh new file mode 100755 index 000000000000..894ff15e69e5 --- /dev/null +++ b/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/hypershift-enable-user-workload-monitoring-commands.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +set -euo pipefail +# No `set -x`: this step does not handle credentials, but it is a sibling of +# steps that do and the surrounding convention is to keep tracing off. + +KUBECONFIG_FILE="${MONITORING_KUBECONFIG:-${SHARED_DIR}/management_cluster_kubeconfig}" +if [[ ! -s "${KUBECONFIG_FILE}" ]]; then + echo "FATAL: ${KUBECONFIG_FILE} is missing or empty." + exit 1 +fi + +k() { oc --kubeconfig="${KUBECONFIG_FILE}" "$@"; } + +# A hosted control plane runs as ordinary workloads in a user namespace of the +# management cluster, so its ServiceMonitors are only scraped when user workload +# monitoring is on. The same is true for any operator shipping a ServiceMonitor +# in a namespace the cluster monitoring stack does not own, which is why this is +# a step of its own rather than part of a single workload. +existing="" +if ! existing="$(k -n openshift-monitoring get configmap cluster-monitoring-config \ + -o jsonpath='{.data.config\.yaml}' 2>/dev/null)"; then + echo "Creating cluster-monitoring-config with user workload monitoring enabled" + k apply -f - <<'EOF' +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-monitoring-config + namespace: openshift-monitoring +data: + config.yaml: | + enableUserWorkload: true +EOF +elif grep -Eq '^[[:space:]]*enableUserWorkload:[[:space:]]*true' <<< "${existing}"; then + echo "User workload monitoring is already enabled" +else + echo "Appending enableUserWorkload to the existing cluster-monitoring-config" + printf '%s\nenableUserWorkload: true\n' "${existing}" > /tmp/cluster-monitoring-config.yaml + k -n openshift-monitoring create configmap cluster-monitoring-config \ + --from-file=config.yaml=/tmp/cluster-monitoring-config.yaml \ + --dry-run=client -o yaml | k replace -f - +fi + +echo "Waiting for the user workload Prometheus StatefulSet to be created" +for _ in $(seq 1 "${UWM_WAIT_ATTEMPTS}"); do + if k -n openshift-user-workload-monitoring get statefulset prometheus-user-workload >/dev/null 2>&1; then + break + fi + sleep 10 +done + +k -n openshift-user-workload-monitoring rollout status \ + statefulset/prometheus-user-workload --timeout="${UWM_ROLLOUT_TIMEOUT}" + +k -n openshift-user-workload-monitoring get pods \ + > "${ARTIFACT_DIR}/user-workload-monitoring-pods.txt" 2>&1 || true diff --git a/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/hypershift-enable-user-workload-monitoring-ref.metadata.json b/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/hypershift-enable-user-workload-monitoring-ref.metadata.json new file mode 100644 index 000000000000..d50d1d4c3dbb --- /dev/null +++ b/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/hypershift-enable-user-workload-monitoring-ref.metadata.json @@ -0,0 +1,21 @@ +{ + "path": "hypershift/enable-user-workload-monitoring/hypershift-enable-user-workload-monitoring-ref.yaml", + "owners": { + "approvers": [ + "bryan-cox", + "cblecker", + "csrwng", + "enxebre", + "jparrill", + "mgencur" + ], + "reviewers": [ + "bryan-cox", + "cblecker", + "csrwng", + "enxebre", + "jparrill", + "mgencur" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/hypershift-enable-user-workload-monitoring-ref.yaml b/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/hypershift-enable-user-workload-monitoring-ref.yaml new file mode 100644 index 000000000000..1e3b85a1e44d --- /dev/null +++ b/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/hypershift-enable-user-workload-monitoring-ref.yaml @@ -0,0 +1,37 @@ +ref: + as: hypershift-enable-user-workload-monitoring + from: cli + env: + - name: MONITORING_KUBECONFIG + default: "" + documentation: |- + Kubeconfig of the cluster to enable user workload monitoring on. Defaults + to ${SHARED_DIR}/management_cluster_kubeconfig, which is where the + HyperShift workflows put the management cluster. Point it elsewhere to + reuse this step on a standalone cluster. + - name: UWM_WAIT_ATTEMPTS + default: "60" + documentation: |- + How many 10 second polls to wait for the prometheus-user-workload + StatefulSet to appear after the ConfigMap is written. + - name: UWM_ROLLOUT_TIMEOUT + default: "10m" + documentation: "Timeout passed to \"oc rollout status\" for prometheus-user-workload." + commands: hypershift-enable-user-workload-monitoring-commands.sh + resources: + requests: + cpu: 100m + memory: 200Mi + timeout: 20m0s + documentation: |- + Enables user workload monitoring on a cluster and waits for the user + workload Prometheus to roll out. + + Anything that runs in a namespace the cluster monitoring stack does not own + -- a hosted control plane in "clusters-", an operator shipping its own + ServiceMonitor -- is only scraped once user workload monitoring is on. Any + step that later queries those series needs this to have happened first, so + it is a step of its own rather than a preamble inside one workload. + + The ConfigMap is merged rather than overwritten, and the step is a no-op when + user workload monitoring is already enabled. diff --git a/ci-operator/step-registry/hypershift/etcd-perf/OWNERS b/ci-operator/step-registry/hypershift/etcd-perf/OWNERS new file mode 120000 index 000000000000..ec405d65a79d --- /dev/null +++ b/ci-operator/step-registry/hypershift/etcd-perf/OWNERS @@ -0,0 +1 @@ +../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/hypershift/etcd-perf/README.md b/ci-operator/step-registry/hypershift/etcd-perf/README.md new file mode 100644 index 000000000000..46db3ff5b6a0 --- /dev/null +++ b/ci-operator/step-registry/hypershift/etcd-perf/README.md @@ -0,0 +1,121 @@ +# hypershift-etcd-perf + +Runs one `kube-burner-ocp etcd-density` workload against a HyperShift hosted +cluster and measures the hosted control plane's etcd while it runs. + +This exists to give etcd sharding a periodic measurement, as opposed to the +`e2e-*-etcd-sharding` jobs, which are functional: they assert that shards get +deployed, that the kube-apiserver is given the right `--etcd-servers-overrides`, +that keys land in the shard they are routed to, and that the shards survive a +restart. Those jobs deliberately measure nothing. This one produces numbers and +nothing else. + +## Shape + +The chain is deliberately three small steps rather than one script, so that the +only HyperShift-specific part is the plumbing and the workload itself is the +perf and scale team's own step, used unmodified: + +| Step | Owner | What it does | +|---|---|---| +| `hypershift-enable-user-workload-monitoring` | HyperShift | Turns on user workload monitoring on the management cluster | +| `hypershift-hcp-metrics-endpoint` | HyperShift | Resolves the management cluster's Thanos, writes the kube-burner endpoint file | +| `openshift-qe-etcd-density-annotation-churn` | perf and scale | Runs the workload | + +Nothing in this directory contains a metrics query. The queries come from +`metrics-profiles/` in `cloud-bulldozer/e2e-benchmarking`, at the same tag the +`openshift-qe` workload steps resolve, so there is one copy of them and it is +owned by the team that owns the measurements. + +## Why a hosted cluster needs the extra plumbing + +On a standalone cluster, kube-burner points at one cluster and finds etcd in +`openshift-etcd` on that same cluster. On a hosted cluster the two halves are on +different clusters: the load lands on the guest API server, but the etcd under +test is a set of pods in `clusters-` on the management cluster. Two things +follow: + +- Those pods are in a user namespace, so nothing scrapes their ServiceMonitors + until user workload monitoring is enabled. +- kube-burner's own Prometheus discovery finds the cluster in `KUBECONFIG`, + which is the guest. The series it needs are on the other one, so it is handed + an explicit endpoint instead. + +`hypershift-hcp-metrics-endpoint` also verifies that etcd series are actually +arriving before the workload starts, so a job cannot burn hours producing load +and no measurement. + +The `kube-burner-ocp-wrapper` in `e2e-benchmarking` does have a +hosted-control-plane mode, but its AWS path is ROSA specific: it derives the +control plane namespace from the `api.openshift.com/name` resource tag and +expects an Observability Operator route on the management cluster. Neither holds +on the self-managed HyperShift CI lanes. Its *profiles* are reused; its +discovery is not. + +## Per-shard breakdown + +Not there yet. The upstream `rosa/hosted-cp-metrics.yml` aggregates etcd series +across the control plane namespace without a `by (pod)` breakdown, so a sharded +run currently reports the shards summed together. Because the pod names carry +the shard name (`etcd-0`, `etcd-events-0`, `etcd-leases-0`, ...), splitting them +is a `by (pod)` away — but that belongs in `e2e-benchmarking` next to the rest +of the queries, not in a copy of the profile here. Until that lands, the shard +topology in effect is recorded in `hostedcluster-etcd-spec.json` and +`etcd-statefulsets.txt` in the artifact directory. + +## Comparing standard against sharded + +The chain does not compare topologies. It measures whichever hosted cluster it +is pointed at. A comparison is two jobs on the same schedule running the same +workload with the same flags, where one creates the hosted cluster with +`HYPERSHIFT_ETCD_SHARDS` set and the other does not. + +The shard topology the jobs use is the one agreed with the perf and scale team: +three PersistentVolume-backed main etcd members, a three-member +PersistentVolume-backed `leases` shard so that cluster heartbeat signals get +their own highly available shard, and a single EmptyDir-backed `events` shard. + +```yaml +HYPERSHIFT_CP_AVAILABILITY_POLICY: HighlyAvailable # the three main members +HYPERSHIFT_ETCD_SHARDS: |- + name=leases,resources=coordination.k8s.io/leases,replicas=3,storage=PersistentVolume + name=events,resources=/events;events.k8s.io/events,replicas=1,storage=EmptyDir +``` + +`EtcdSharding` is a `TechPreviewNoUpgrade` management cluster feature gate, so +the sharded job also needs `TECH_PREVIEW_NO_UPGRADE: "true"`. + +## Workloads + +The four `etcd-density` sub-workloads are the ones the OpenShift Performance and +Scale team used to characterise standard against sharded on HCP: + +| Workload | What it stresses | `openshift-qe` step | +|---|---|---| +| `annotation-churn` | Revision bloat from repeated annotation patching | yes | +| `db-quota-pressure` | Large object fill driving the backend toward its storage quota | yes | +| `crashloop-flood` | Continuous event pressure through the PATCH/count-increment path, which is the shape most real event storms take | not yet | +| `event-storm` | Bulk event creation against a concurrent victim workload | not yet | + +They are added one per pull request, starting with `annotation-churn`. Adding +one is swapping the last ref of the chain, plus writing the `openshift-qe` step +for it if there is not one yet. + +`db-quota-pressure` needs `GC: "false"`, otherwise kube-burner removes the +objects that are supposed to be holding the backend under pressure. Its +`openshift-qe` step already defaults it that way. + +## Results + +`collected-metrics-/` in the artifact directory. + +Not Elasticsearch, yet, and so not Orion. `run.sh` in `e2e-benchmarking` passes +either `--metrics-endpoint` or `--es-server` to kube-burner, never both, and the +hosted control plane series only exist on the endpoint. Its existing +hosted-control-plane mode would resolve that, but it only knows ROSA and ARO; +giving it a self-managed HyperShift branch is what unblocks indexing, and it +belongs in that repo. Until then these jobs are read from their artifacts. + +No credential handling lives in this directory. The one Elasticsearch password +in the chain is read by the `openshift-qe` workload step, which already owned +that code before this chain existed. diff --git a/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-chain.metadata.json b/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-chain.metadata.json new file mode 100644 index 000000000000..7a1a76aac3bb --- /dev/null +++ b/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-chain.metadata.json @@ -0,0 +1,21 @@ +{ + "path": "hypershift/etcd-perf/hypershift-etcd-perf-chain.yaml", + "owners": { + "approvers": [ + "bryan-cox", + "cblecker", + "csrwng", + "enxebre", + "jparrill", + "mgencur" + ], + "reviewers": [ + "bryan-cox", + "cblecker", + "csrwng", + "enxebre", + "jparrill", + "mgencur" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-chain.yaml b/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-chain.yaml new file mode 100644 index 000000000000..cc2732c016f5 --- /dev/null +++ b/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-chain.yaml @@ -0,0 +1,34 @@ +chain: + as: hypershift-etcd-perf + steps: + - ref: hypershift-enable-user-workload-monitoring + - ref: hypershift-hcp-metrics-endpoint + - ref: openshift-qe-etcd-density-annotation-churn + documentation: |- + Runs a kube-burner-ocp "etcd-density" workload against a HyperShift hosted + cluster and collects the hosted control plane's etcd and kube-apiserver + metrics from the management cluster. + + The chain is three steps because each of them is useful on its own: + + - hypershift-enable-user-workload-monitoring turns on the monitoring the + hosted control plane namespace needs to be scraped at all. + - hypershift-hcp-metrics-endpoint points kube-burner at the management + cluster's Thanos, with the query set taken from e2e-benchmarking. + - openshift-qe-etcd-density-annotation-churn is the perf and scale team's + own workload step, used unmodified, so the HCP numbers and the standalone + OpenShift numbers come from the same workload definition. + + Shard topologies are not compared by the chain. Comparison is done by + running it twice, once against a hosted cluster created with + HYPERSHIFT_ETCD_SHARDS and once without, and diffing the two result sets. + + Other etcd-density variants (crashloop-flood, db-quota-pressure, + event-storm) are added by swapping the last ref, one variant per pull + request. db-quota-pressure already has an openshift-qe step; the other two + do not yet. + + Requires a hosted cluster to already exist, i.e. the hypershift-aws-create + chain or an equivalent must have run and written + ${SHARED_DIR}/cluster-name, ${SHARED_DIR}/nested_kubeconfig and + ${SHARED_DIR}/management_cluster_kubeconfig. diff --git a/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/OWNERS b/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/OWNERS new file mode 120000 index 000000000000..ec405d65a79d --- /dev/null +++ b/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/OWNERS @@ -0,0 +1 @@ +../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/hypershift-hcp-metrics-endpoint-commands.sh b/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/hypershift-hcp-metrics-endpoint-commands.sh new file mode 100755 index 000000000000..12319dc8d26f --- /dev/null +++ b/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/hypershift-hcp-metrics-endpoint-commands.sh @@ -0,0 +1,155 @@ +#!/bin/bash + +set -euo pipefail +# Do not enable `set -x`. This step mints a Prometheus bearer token; tracing +# would put it in the CI logs. + +MC_KUBECONFIG="${SHARED_DIR}/management_cluster_kubeconfig" +for f in "${MC_KUBECONFIG}" "${SHARED_DIR}/cluster-name"; do + if [[ ! -s "${f}" ]]; then + echo "FATAL: ${f} is missing or empty; this step needs a hosted cluster to already exist." + exit 1 + fi +done + +CLUSTER_NAME="$(cat "${SHARED_DIR}/cluster-name")" +# HyperShift puts the hosted control plane in "-", +# and every CI workflow that feeds this step creates the HostedCluster in "clusters". +HCP_NAMESPACE="clusters-${CLUSTER_NAME}" + +mc() { oc --kubeconfig="${MC_KUBECONFIG}" "$@"; } + +echo "Hosted cluster: ${CLUSTER_NAME}" +echo "Hosted control plane ns: ${HCP_NAMESPACE}" + +if ! mc get namespace "${HCP_NAMESPACE}" >/dev/null 2>&1; then + echo "FATAL: namespace ${HCP_NAMESPACE} does not exist on the management cluster." + exit 1 +fi + +# Record which etcd topology produced the results. With sharding the pod names +# carry the shard name (etcd-0, etcd-events-0, ...), so this is also the record +# of which shards the metrics below are coming from. +mc get statefulset -n "${HCP_NAMESPACE}" -l app=etcd -o wide 2>&1 \ + | tee "${ARTIFACT_DIR}/etcd-statefulsets.txt" || true +mc get hostedcluster -n clusters "${CLUSTER_NAME}" -o jsonpath='{.spec.etcd}' \ + > "${ARTIFACT_DIR}/hostedcluster-etcd-spec.json" 2>/dev/null || true + +# --------------------------------------------------------------------------- +# Metrics profiles +# +# Taken from the kube-burner-ocp wrapper in e2e-benchmarking rather than +# written here, so the queries stay owned by the perf and scale team and this +# repo does not carry a second copy that drifts. The profiles are copied into +# SHARED_DIR because /tmp does not survive into the step that runs the +# workload. +# --------------------------------------------------------------------------- +REPO_URL="https://github.com/cloud-bulldozer/e2e-benchmarking" +if [[ "${E2E_VERSION}" == "default" ]]; then + E2E_REF="$(git ls-remote --tags "${REPO_URL}.git" \ + | awk -F'refs/tags/' '{print $2}' | grep -v '\^{}' | sort -V | tail -n1)" +else + E2E_REF="${E2E_VERSION}" +fi +echo "Using e2e-benchmarking ${E2E_REF} for the metrics profiles" +git clone "${REPO_URL}" --branch "${E2E_REF}" --depth 1 /tmp/e2e-benchmarking + +WRAPPER=/tmp/e2e-benchmarking/workloads/kube-burner-ocp-wrapper +PROFILE_DIR="${SHARED_DIR}/hcp-metrics-profiles" +mkdir -p "${PROFILE_DIR}" + +for profile in ${HCP_METRICS_PROFILES}; do + src="${WRAPPER}/metrics-profiles/${profile}" + if [[ ! -s "${src}" ]]; then + echo "FATAL: ${profile} does not exist in e2e-benchmarking ${E2E_REF}." + exit 1 + fi + # The upstream profiles are written for ROSA, where the hosted control plane + # namespace is "-" and the queries match it with + # namespace=~".+{{.HCP_NAMESPACE}}". Self-managed HyperShift uses + # "clusters-", so binding the placeholder to the cluster name keeps the + # upstream regex correct without editing the queries. + sed "s|{{\.HCP_NAMESPACE}}|${CLUSTER_NAME}|g" "${src}" > "${PROFILE_DIR}/$(basename "${profile}")" +done +cp -r "${PROFILE_DIR}" "${ARTIFACT_DIR}/hcp-metrics-profiles" + +# --------------------------------------------------------------------------- +# Thanos endpoint and token +# +# thanos-querier rather than prometheus-k8s: the hosted control plane series are +# collected by the user workload Prometheus, and thanos-querier is the endpoint +# that serves both stacks. +# --------------------------------------------------------------------------- +PROM_SA="hypershift-hcp-metrics" +mc -n openshift-monitoring create serviceaccount "${PROM_SA}" --dry-run=client -o yaml | mc apply -f - +mc adm policy add-cluster-role-to-user cluster-monitoring-view \ + -z "${PROM_SA}" -n openshift-monitoring >/dev/null + +THANOS_HOST="$(mc -n openshift-monitoring get route thanos-querier -o jsonpath='{.spec.host}')" +if [[ -z "${THANOS_HOST}" ]]; then + echo "FATAL: could not resolve the thanos-querier route on the management cluster." + exit 1 +fi +THANOS_URL="https://${THANOS_HOST}" +PROM_TOKEN="$(mc -n openshift-monitoring create token "${PROM_SA}" --duration="${PROM_TOKEN_DURATION}")" +echo "Thanos endpoint: ${THANOS_URL}" + +# The management cluster's ingress certificate is not one this container trusts, +# hence -k here and skipTLSVerify in the endpoint file below. +promq() { + curl --fail --silent --show-error -k --get --data-urlencode "query=$1" \ + -H "Authorization: Bearer ${PROM_TOKEN}" \ + "${THANOS_URL}/api/v1/query" +} + +echo "Waiting ${SCRAPE_SETTLE_SECONDS}s for the hosted control plane targets to be scraped" +sleep "${SCRAPE_SETTLE_SECONDS}" + +# Fail now rather than after a multi-hour workload that produced no measurement. +# A query that errors, returns no result, or returns zero members all mean the +# same thing here: there is nothing to measure. +if ! response="$(promq "count(etcd_server_has_leader{namespace=\"${HCP_NAMESPACE}\"})")"; then + echo "FATAL: the Thanos query failed. The endpoint or the token is wrong, or" + echo "thanos-querier on the management cluster is not serving queries." + exit 1 +fi +series_count="$(jq -r '.data.result[0].value[1] // empty' <<< "${response}")" +if [[ -z "${series_count}" || "${series_count}" == "0" ]]; then + echo "FATAL: Thanos returns no etcd metrics for namespace ${HCP_NAMESPACE}." + echo "Check that the hosted control plane's etcd ServiceMonitors exist and that" + echo "user workload monitoring came up (hypershift-enable-user-workload-monitoring)." + mc get servicemonitor -n "${HCP_NAMESPACE}" 2>&1 | head -20 || true + exit 1 +fi +echo "Scraping ${series_count} etcd member(s) in ${HCP_NAMESPACE}" + +# --------------------------------------------------------------------------- +# Endpoint file +# +# Written to SHARED_DIR, never to ARTIFACT_DIR: it carries the bearer token. +# The indexer is local so the series land in the workload step's working +# directory and from there in the artifacts; indexing to Elasticsearch stays +# with the openshift-qe workload step, which already owns those credentials. +# --------------------------------------------------------------------------- +ENDPOINT_FILE="${SHARED_DIR}/hcp-metrics-endpoint.yml" +{ + echo "- endpoint: ${THANOS_URL}" + echo " token: ${PROM_TOKEN}" + echo " skipTLSVerify: true" + echo " step: ${METRICS_STEP}" + echo " metrics:" + for profile in ${HCP_METRICS_PROFILES}; do + echo " - ${PROFILE_DIR}/$(basename "${profile}")" + done + echo " indexer:" + echo " type: local" + # kube-burner renders the endpoint file as a template over the environment, + # the same way the upstream metrics-endpoint.yml uses {{.ES_SERVER}}. UUID is + # exported by the openshift-qe workload step, and the directory name is the + # one that step copies into ARTIFACT_DIR when the run finishes. + echo " metricsDirectory: collected-metrics-{{.UUID}}" +} > "${ENDPOINT_FILE}" +chmod 600 "${ENDPOINT_FILE}" + +echo "${HCP_NAMESPACE}" > "${SHARED_DIR}/hcp-namespace" +echo "Wrote ${ENDPOINT_FILE}" diff --git a/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/hypershift-hcp-metrics-endpoint-ref.metadata.json b/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/hypershift-hcp-metrics-endpoint-ref.metadata.json new file mode 100644 index 000000000000..2fbf1061e83c --- /dev/null +++ b/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/hypershift-hcp-metrics-endpoint-ref.metadata.json @@ -0,0 +1,21 @@ +{ + "path": "hypershift/hcp-metrics-endpoint/hypershift-hcp-metrics-endpoint-ref.yaml", + "owners": { + "approvers": [ + "bryan-cox", + "cblecker", + "csrwng", + "enxebre", + "jparrill", + "mgencur" + ], + "reviewers": [ + "bryan-cox", + "cblecker", + "csrwng", + "enxebre", + "jparrill", + "mgencur" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/hypershift-hcp-metrics-endpoint-ref.yaml b/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/hypershift-hcp-metrics-endpoint-ref.yaml new file mode 100644 index 000000000000..e3f448ec21e9 --- /dev/null +++ b/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/hypershift-hcp-metrics-endpoint-ref.yaml @@ -0,0 +1,71 @@ +ref: + as: hypershift-hcp-metrics-endpoint + # Same image as the openshift-qe workload steps, so the e2e-benchmarking + # checkout this step does and the one the workload step does are identical. + from_image: + namespace: ci + name: ocp-qe-perfscale-ci + tag: latest + cli: latest + env: + - name: E2E_VERSION + default: "default" + documentation: |- + e2e-benchmarking tag to take the metrics profiles from. "default" resolves + to the latest tag, which is what the openshift-qe workload steps do, so + the profiles and the wrapper that consumes them stay on the same release. + - name: HCP_METRICS_PROFILES + default: "rosa/hosted-cp-metrics.yml" + documentation: |- + Space separated list of metrics profiles to take from the kube-burner-ocp + wrapper's "metrics-profiles" directory in e2e-benchmarking. The default is + the hosted control plane profile: etcd, kube-apiserver and OVN series for + the control plane namespace. The "rosa" directory is the hosted control + plane profile for OpenShift-flavoured management clusters; self-managed + HyperShift reuses it, the difference between the two products is the + namespace layout, which this step binds for you. + - name: METRICS_STEP + default: "30s" + documentation: "Prometheus range query resolution used when collecting the profiles." + - name: SCRAPE_SETTLE_SECONDS + default: "180" + documentation: |- + How long to wait before checking that the hosted control plane etcd + targets are actually being scraped. Enabling user workload monitoring and + the first scrape of a newly created namespace are not instantaneous. + - name: PROM_TOKEN_DURATION + default: "8h" + documentation: |- + Lifetime of the Thanos bearer token. Must outlast the workload step that + consumes the endpoint file. + commands: hypershift-hcp-metrics-endpoint-commands.sh + resources: + requests: + cpu: 100m + memory: 200Mi + timeout: 30m0s + documentation: |- + Prepares ${SHARED_DIR}/hcp-metrics-endpoint.yml so that a kube-burner + workload driven against a hosted cluster collects its control plane metrics + from the management cluster. + + A hosted cluster is split: the workload is applied to the guest API server, + but the etcd and kube-apiserver being measured run as pods in the + "clusters-" namespace of the management cluster. kube-burner's own + Prometheus discovery finds the cluster in KUBECONFIG, which is the wrong one + of the two, so this step resolves the management cluster's thanos-querier + route, mints a cluster-monitoring-view token for it, and writes an endpoint + file the workload step picks up. + + The queries are not written here. They are copied from the kube-burner-ocp + wrapper in cloud-bulldozer/e2e-benchmarking, the same source and the same + tag the openshift-qe workload steps use, with the hosted control plane + namespace bound for self-managed HyperShift. Changes to what is measured + belong in that repo. + + Requires hypershift-enable-user-workload-monitoring to have run, and a + hosted cluster to exist: ${SHARED_DIR}/cluster-name and + ${SHARED_DIR}/management_cluster_kubeconfig must be present. + + The endpoint file carries a bearer token. It is written to SHARED_DIR with + mode 600 and is never copied to ARTIFACT_DIR. diff --git a/ci-operator/step-registry/openshift-qe/etcd-density-annotation-churn/openshift-qe-etcd-density-annotation-churn-commands.sh b/ci-operator/step-registry/openshift-qe/etcd-density-annotation-churn/openshift-qe-etcd-density-annotation-churn-commands.sh index d83a6bf37dc9..a354a4518058 100755 --- a/ci-operator/step-registry/openshift-qe/etcd-density-annotation-churn/openshift-qe-etcd-density-annotation-churn-commands.sh +++ b/ci-operator/step-registry/openshift-qe/etcd-density-annotation-churn/openshift-qe-etcd-density-annotation-churn-commands.sh @@ -5,6 +5,22 @@ set -o pipefail # Do not enable `set -x`: these scripts read and export Elasticsearch # credentials (ES_USERNAME/ES_PASSWORD) and construct an authenticated # ES_SERVER URL; tracing would leak them into the CI logs. + +# Self-managed HyperShift. Both files are written by the hypershift-etcd-perf +# chain and absent everywhere else, so this is a no-op on a standalone cluster. +# +# A hosted cluster is split across two clusters: the workload is applied to the +# guest API server, but the etcd and kube-apiserver being measured are pods on +# the management cluster. So the load goes to nested_kubeconfig while the +# metrics come from the endpoint file hypershift-hcp-metrics-endpoint prepared. +HCP_METRICS_ENDPOINT="${SHARED_DIR}/hcp-metrics-endpoint.yml" +HOSTED_CLUSTER=false +if [[ -s "${SHARED_DIR}/nested_kubeconfig" && -s "${HCP_METRICS_ENDPOINT}" ]]; then + HOSTED_CLUSTER=true + echo "Hosted cluster detected; driving the workload against the guest API server" + export KUBECONFIG="${SHARED_DIR}/nested_kubeconfig" +fi + oc projects oc version pushd /tmp @@ -32,7 +48,12 @@ sed -i '/index\.sh/s|WORKLOAD="$WORKLOAD"|WORKLOAD="annotation-churn"|' run.sh EXTRA_FLAGS="--iterations=${ANNOTATION_CHURN_ITERATIONS} --patch-replicas=${PATCH_REPLICAS} --patch-rounds=${PATCH_ROUNDS}" # Flags not handled by run.sh EXTRA_FLAGS+=" --gc-metrics=${GC_METRICS} --profile-type=${PROFILE_TYPE}" -EXTRA_FLAGS+=" --metrics-profile=${METRICS_PROFILES}" +if [[ "${HOSTED_CLUSTER}" == "true" ]]; then + # The endpoint file names its own profiles, so --metrics-profile does not apply. + EXTRA_FLAGS+=" --metrics-endpoint=${HCP_METRICS_ENDPOINT}" +else + EXTRA_FLAGS+=" --metrics-profile=${METRICS_PROFILES}" +fi if [[ "${ENABLE_LOCAL_INDEX}" == "true" ]]; then EXTRA_FLAGS+=" --local-indexing" fi @@ -43,6 +64,17 @@ fi EXTRA_FLAGS+=" ${KB_FLAGS}" export ES_SERVER="https://$ES_USERNAME:$ES_PASSWORD@$ES_HOST" +if [[ "${HOSTED_CLUSTER}" == "true" ]]; then + # run.sh adds --es-server only when it is not already passing a metrics + # endpoint, and the two are mutually exclusive there. Leaving ES_SERVER set + # would have it add both. The endpoint file indexes locally instead, and + # index.sh exits cleanly when ES_SERVER is empty. + # + # Indexing hosted control plane series to Elasticsearch, and so Orion + # regression detection over them, needs run.sh to grow a self-managed + # HyperShift branch next to its existing ROSA and ARO ones. + export ES_SERVER="" +fi export EXTRA_FLAGS UUID ADDITIONAL_PARAMS set +o errexit