From 2d8c23dabb36d93f156675364da33984a876574f Mon Sep 17 00:00:00 2001 From: mehabhalodiya Date: Tue, 29 Sep 2026 15:06:14 +0100 Subject: [PATCH 1/2] Add etcd sharding periodics for hypershift main Signed-off-by: mehabhalodiya --- .../hypershift/openshift-hypershift-main.yaml | 65 ++ ...-hypershift-main__periodics-etcd-perf.yaml | 115 ++++ .../openshift-hypershift-main-periodics.yaml | 630 ++++++++++++++++++ .../openshift-hypershift-main-presubmits.yaml | 428 ++++++++++++ .../hypershift-aws-create-guests-commands.sh | 4 + .../hypershift-aws-create-guests-ref.yaml | 3 + .../create/hypershift-aws-create-chain.yaml | 17 + .../hypershift-aws-run-e2e-v2-chain.yaml | 7 + ...zure-create-selfmanaged-guests-commands.sh | 4 + ...t-azure-create-selfmanaged-guests-ref.yaml | 3 + ...ft-azure-run-e2e-v2-selfmanaged-chain.yaml | 6 + .../step-registry/hypershift/etcd-perf/OWNERS | 15 + .../hypershift/etcd-perf/README.md | 78 +++ .../hypershift-etcd-perf-commands.sh | 450 +++++++++++++ .../hypershift-etcd-perf-ref.metadata.json | 21 + .../etcd-perf/hypershift-etcd-perf-ref.yaml | 112 ++++ 16 files changed, 1958 insertions(+) create mode 100644 ci-operator/config/openshift/hypershift/openshift-hypershift-main__periodics-etcd-perf.yaml create mode 100644 ci-operator/step-registry/hypershift/etcd-perf/OWNERS create mode 100644 ci-operator/step-registry/hypershift/etcd-perf/README.md create mode 100644 ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-commands.sh create mode 100644 ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-ref.metadata.json create mode 100644 ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-ref.yaml diff --git a/ci-operator/config/openshift/hypershift/openshift-hypershift-main.yaml b/ci-operator/config/openshift/hypershift/openshift-hypershift-main.yaml index 5e904576c36e..55b06f135760 100644 --- a/ci-operator/config/openshift/hypershift/openshift-hypershift-main.yaml +++ b/ci-operator/config/openshift/hypershift/openshift-hypershift-main.yaml @@ -296,6 +296,47 @@ tests: TECH_PREVIEW_NO_UPGRADE: "true" TEST_SUITE: openshift/conformance/parallel/minimal workflow: hypershift-aws-e2e-backuprestore +- always_run: false + as: e2e-v2-aws-etcd-sharding + capabilities: + - arm64 + optional: true + steps: + cluster_profile: hypershift-aws + env: + NODEPOOL_N1_VERSION: "5.0" + TECH_PREVIEW_NO_UPGRADE: "true" + TEST_PLAN_FILE: /hypershift/testplans/aws-etcd-sharding.yaml + workflow: hypershift-aws-e2e-v2 +- always_run: false + as: e2e-v2-azure-self-managed-etcd-sharding + capabilities: + - arm64 + optional: true + steps: + cluster_profile: hypershift-azure + env: + ENABLE_HYPERSHIFT_CERT_ROTATION_SCALE: "true" + HYPERSHIFT_AZURE_LOCATION: centralus + NODEPOOL_N1_VERSION: "5.0" + TECH_PREVIEW_NO_UPGRADE: "true" + TEST_PLAN_FILE: /hypershift/testplans/azure-etcd-sharding.yaml + workflow: hypershift-azure-e2e-v2-self-managed +- always_run: false + as: e2e-v2-aws-backuprestore-etcd-sharding + capabilities: + - arm64 + optional: true + steps: + cluster_profile: hypershift-aws + env: + HYPERSHIFT_CP_AVAILABILITY_POLICY: HighlyAvailable + HYPERSHIFT_ETCD_SHARDS: |- + name=events,resources=/events;events.k8s.io/events,replicas=3,storage=EmptyDir + name=leases,resources=coordination.k8s.io/leases,replicas=3,storage=PersistentVolume + TECH_PREVIEW_NO_UPGRADE: "true" + TEST_SUITE: openshift/conformance/parallel/minimal + workflow: hypershift-aws-e2e-backuprestore - always_run: false as: e2e-kubevirt-metal-ovn-backuprestore capabilities: @@ -795,6 +836,30 @@ tests: helmValues: controlPlaneOperatorImage: IMAGE_REPO:IMAGE_TAG workflow: rosa-seaplane-ephemeral-e2e +- as: e2e-aws-etcd-sharding + capabilities: + - arm64 + cron: 15 3 * * * + steps: + cluster_profile: hypershift-aws + env: + NODEPOOL_N1_VERSION: "5.0" + TECH_PREVIEW_NO_UPGRADE: "true" + TEST_PLAN_FILE: /hypershift/testplans/aws-etcd-sharding.yaml + workflow: hypershift-aws-e2e-v2 +- as: e2e-azure-etcd-sharding + capabilities: + - arm64 + cron: 15 5 * * 3 + steps: + cluster_profile: hypershift-azure + env: + ENABLE_HYPERSHIFT_CERT_ROTATION_SCALE: "true" + HYPERSHIFT_AZURE_LOCATION: centralus + NODEPOOL_N1_VERSION: "5.0" + TECH_PREVIEW_NO_UPGRADE: "true" + TEST_PLAN_FILE: /hypershift/testplans/azure-etcd-sharding.yaml + workflow: hypershift-azure-e2e-v2-self-managed zz_generated_metadata: branch: main org: openshift diff --git a/ci-operator/config/openshift/hypershift/openshift-hypershift-main__periodics-etcd-perf.yaml b/ci-operator/config/openshift/hypershift/openshift-hypershift-main__periodics-etcd-perf.yaml new file mode 100644 index 000000000000..f4198111d6fe --- /dev/null +++ b/ci-operator/config/openshift/hypershift/openshift-hypershift-main__periodics-etcd-perf.yaml @@ -0,0 +1,115 @@ +base_images: + hypershift-operator: + name: hypershift-operator + namespace: hypershift + tag: latest + hypershift-tests: + name: hypershift-tests + namespace: hypershift + tag: latest + upi-installer: + name: "5.1" + namespace: ocp + tag: upi-installer +prowgen: + enable_secrets_store_csi_driver: true +releases: + initial: + candidate: + product: ocp + stream: ci + version: "5.1" + latest: + candidate: + product: ocp + stream: ci + version: "5.1" +resources: + '*': + requests: + cpu: 100m + memory: 200Mi +tests: +- as: event-storm-standard + cron: 20 7 * * 2 + steps: + cluster_profile: hypershift-aws + env: + ETCD_MAX_LEADER_CHANGES: "5" + ETCD_MAX_P99_COMMIT_LATENCY_MS: "500" + ETCD_PERF_FLAGS: --iterations=60 --pod-replicas=2 --event-iterations=2000 --event-replicas=100 + --storm-workers=20 + ETCD_PERF_VARIANT: standard + ETCD_PERF_WORKLOAD: event-storm + HYPERSHIFT_CP_AVAILABILITY_POLICY: HighlyAvailable + HYPERSHIFT_INSTANCE_TYPE: m5.xlarge + HYPERSHIFT_NODE_COUNT: "6" + test: + - ref: hypershift-etcd-perf + workflow: hypershift-aws-e2e-cluster + timeout: 6h0m0s +- as: event-storm-sharded + cron: 20 13 * * 2 + steps: + cluster_profile: hypershift-aws + env: + ETCD_MAX_LEADER_CHANGES: "5" + ETCD_MAX_P99_COMMIT_LATENCY_MS: "500" + ETCD_PERF_FLAGS: --iterations=60 --pod-replicas=2 --event-iterations=2000 --event-replicas=100 + --storm-workers=20 + ETCD_PERF_VARIANT: sharded + ETCD_PERF_WORKLOAD: event-storm + HYPERSHIFT_CP_AVAILABILITY_POLICY: HighlyAvailable + HYPERSHIFT_ETCD_SHARDS: name=events,resources=/events;events.k8s.io/events,replicas=3,storage=PersistentVolume + HYPERSHIFT_INSTANCE_TYPE: m5.xlarge + HYPERSHIFT_NODE_COUNT: "6" + TECH_PREVIEW_NO_UPGRADE: "true" + test: + - ref: hypershift-etcd-perf + workflow: hypershift-aws-e2e-cluster + timeout: 6h0m0s +- as: db-quota-pressure-standard + cron: 20 7 * * 4 + steps: + cluster_profile: hypershift-aws + env: + ETCD_MAX_LEADER_CHANGES: "5" + ETCD_MAX_P99_COMMIT_LATENCY_MS: "500" + ETCD_PERF_FLAGS: --iterations=2000 --iterations-per-namespace=10 --kb-chunks=8 + --kb-size=100 + ETCD_PERF_VARIANT: standard + ETCD_PERF_WORKLOAD: db-quota-pressure + GC: "false" + HYPERSHIFT_CP_AVAILABILITY_POLICY: HighlyAvailable + HYPERSHIFT_INSTANCE_TYPE: m5.xlarge + HYPERSHIFT_NODE_COUNT: "3" + test: + - ref: hypershift-etcd-perf + workflow: hypershift-aws-e2e-cluster + timeout: 6h0m0s +- as: db-quota-pressure-sharded + cron: 20 13 * * 4 + steps: + cluster_profile: hypershift-aws + env: + ETCD_MAX_LEADER_CHANGES: "5" + ETCD_MAX_P99_COMMIT_LATENCY_MS: "500" + ETCD_PERF_FLAGS: --iterations=2000 --iterations-per-namespace=10 --kb-chunks=8 + --kb-size=100 + ETCD_PERF_VARIANT: sharded + ETCD_PERF_WORKLOAD: db-quota-pressure + GC: "false" + HYPERSHIFT_CP_AVAILABILITY_POLICY: HighlyAvailable + HYPERSHIFT_ETCD_SHARDS: name=events,resources=/events;events.k8s.io/events,replicas=3,storage=PersistentVolume + HYPERSHIFT_INSTANCE_TYPE: m5.xlarge + HYPERSHIFT_NODE_COUNT: "3" + TECH_PREVIEW_NO_UPGRADE: "true" + test: + - ref: hypershift-etcd-perf + workflow: hypershift-aws-e2e-cluster + timeout: 6h0m0s +zz_generated_metadata: + branch: main + org: openshift + repo: hypershift + variant: periodics-etcd-perf diff --git a/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-periodics.yaml b/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-periodics.yaml index eb07e50180f9..dc8253b864f3 100644 --- a/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-periodics.yaml +++ b/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-periodics.yaml @@ -298,6 +298,224 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build07 + cron: 15 3 * * * + decorate: true + decoration_config: + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile + - Dockerfile.control-plane + - Dockerfile.e2e + extra_refs: + - base_ref: main + org: openshift + repo: hypershift + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile + - Dockerfile.control-plane + - Dockerfile.e2e + labels: + capability/arm64: arm64 + ci-operator.openshift.io/cloud: hypershift-aws + ci-operator.openshift.io/cloud-cluster-profile: hypershift-aws + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-hypershift-main-e2e-aws-etcd-sharding + spec: + containers: + - args: + - --enable-secrets-store-csi-driver=true + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --gsm-config=/etc/gsm-config/gsm-config.yaml + - --gsm-credentials-file=/etc/gsm-credentials/key.json + - --gsm-project-config=/etc/gsm-config/gsm-project-config.yaml + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=e2e-aws-etcd-sharding + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/gsm-config + name: gsm-config + readOnly: true + - mountPath: /etc/gsm-credentials + name: gsm-sa-key + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - configMap: + name: gsm-config + name: gsm-config + - csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: ci-operator-sa-key-spc + name: gsm-sa-key + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator +- agent: kubernetes + cluster: build07 + cron: 15 5 * * 3 + decorate: true + decoration_config: + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile + - Dockerfile.control-plane + - Dockerfile.e2e + extra_refs: + - base_ref: main + org: openshift + repo: hypershift + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile + - Dockerfile.control-plane + - Dockerfile.e2e + labels: + capability/arm64: arm64 + ci-operator.openshift.io/cloud: hypershift-azure + ci-operator.openshift.io/cloud-cluster-profile: hypershift-azure + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-hypershift-main-e2e-azure-etcd-sharding + spec: + containers: + - args: + - --enable-secrets-store-csi-driver=true + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --gsm-config=/etc/gsm-config/gsm-config.yaml + - --gsm-credentials-file=/etc/gsm-credentials/key.json + - --gsm-project-config=/etc/gsm-config/gsm-project-config.yaml + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=e2e-azure-etcd-sharding + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/gsm-config + name: gsm-config + readOnly: true + - mountPath: /etc/gsm-credentials + name: gsm-sa-key + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - configMap: + name: gsm-config + name: gsm-config + - csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: ci-operator-sa-key-spc + name: gsm-sa-key + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator - agent: kubernetes cluster: build07 cron: '*/30 * * * *' @@ -602,6 +820,418 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build05 + cron: 20 13 * * 4 + decorate: true + decoration_config: + skip_cloning: true + timeout: 6h0m0s + extra_refs: + - base_ref: main + org: openshift + repo: hypershift + labels: + ci-operator.openshift.io/cloud: hypershift-aws + ci-operator.openshift.io/cloud-cluster-profile: hypershift-aws + ci-operator.openshift.io/variant: periodics-etcd-perf + ci.openshift.io/generator: prowgen + job-release: "5.1" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-hypershift-main-periodics-etcd-perf-db-quota-pressure-sharded + spec: + containers: + - args: + - --enable-secrets-store-csi-driver=true + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --gsm-config=/etc/gsm-config/gsm-config.yaml + - --gsm-credentials-file=/etc/gsm-credentials/key.json + - --gsm-project-config=/etc/gsm-config/gsm-project-config.yaml + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=db-quota-pressure-sharded + - --variant=periodics-etcd-perf + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/gsm-config + name: gsm-config + readOnly: true + - mountPath: /etc/gsm-credentials + name: gsm-sa-key + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - configMap: + name: gsm-config + name: gsm-config + - csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: ci-operator-sa-key-spc + name: gsm-sa-key + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator +- agent: kubernetes + cluster: build05 + cron: 20 7 * * 4 + decorate: true + decoration_config: + skip_cloning: true + timeout: 6h0m0s + extra_refs: + - base_ref: main + org: openshift + repo: hypershift + labels: + ci-operator.openshift.io/cloud: hypershift-aws + ci-operator.openshift.io/cloud-cluster-profile: hypershift-aws + ci-operator.openshift.io/variant: periodics-etcd-perf + ci.openshift.io/generator: prowgen + job-release: "5.1" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-hypershift-main-periodics-etcd-perf-db-quota-pressure-standard + spec: + containers: + - args: + - --enable-secrets-store-csi-driver=true + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --gsm-config=/etc/gsm-config/gsm-config.yaml + - --gsm-credentials-file=/etc/gsm-credentials/key.json + - --gsm-project-config=/etc/gsm-config/gsm-project-config.yaml + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=db-quota-pressure-standard + - --variant=periodics-etcd-perf + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/gsm-config + name: gsm-config + readOnly: true + - mountPath: /etc/gsm-credentials + name: gsm-sa-key + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - configMap: + name: gsm-config + name: gsm-config + - csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: ci-operator-sa-key-spc + name: gsm-sa-key + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator +- agent: kubernetes + cluster: build05 + cron: 20 13 * * 2 + decorate: true + decoration_config: + skip_cloning: true + timeout: 6h0m0s + extra_refs: + - base_ref: main + org: openshift + repo: hypershift + labels: + ci-operator.openshift.io/cloud: hypershift-aws + ci-operator.openshift.io/cloud-cluster-profile: hypershift-aws + ci-operator.openshift.io/variant: periodics-etcd-perf + ci.openshift.io/generator: prowgen + job-release: "5.1" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-hypershift-main-periodics-etcd-perf-event-storm-sharded + spec: + containers: + - args: + - --enable-secrets-store-csi-driver=true + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --gsm-config=/etc/gsm-config/gsm-config.yaml + - --gsm-credentials-file=/etc/gsm-credentials/key.json + - --gsm-project-config=/etc/gsm-config/gsm-project-config.yaml + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=event-storm-sharded + - --variant=periodics-etcd-perf + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/gsm-config + name: gsm-config + readOnly: true + - mountPath: /etc/gsm-credentials + name: gsm-sa-key + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - configMap: + name: gsm-config + name: gsm-config + - csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: ci-operator-sa-key-spc + name: gsm-sa-key + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator +- agent: kubernetes + cluster: build05 + cron: 20 7 * * 2 + decorate: true + decoration_config: + skip_cloning: true + timeout: 6h0m0s + extra_refs: + - base_ref: main + org: openshift + repo: hypershift + labels: + ci-operator.openshift.io/cloud: hypershift-aws + ci-operator.openshift.io/cloud-cluster-profile: hypershift-aws + ci-operator.openshift.io/variant: periodics-etcd-perf + ci.openshift.io/generator: prowgen + job-release: "5.1" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-hypershift-main-periodics-etcd-perf-event-storm-standard + spec: + containers: + - args: + - --enable-secrets-store-csi-driver=true + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --gsm-config=/etc/gsm-config/gsm-config.yaml + - --gsm-credentials-file=/etc/gsm-credentials/key.json + - --gsm-project-config=/etc/gsm-config/gsm-project-config.yaml + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=event-storm-standard + - --variant=periodics-etcd-perf + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/gsm-config + name: gsm-config + readOnly: true + - mountPath: /etc/gsm-credentials + name: gsm-sa-key + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - configMap: + name: gsm-config + name: gsm-config + - csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: ci-operator-sa-key-spc + name: gsm-sa-key + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator - agent: kubernetes cluster: build05 cron: 0 8 * * * diff --git a/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-presubmits.yaml b/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-presubmits.yaml index aa921b6dc52a..39f6b9330f31 100644 --- a/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-presubmits.yaml +++ b/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-presubmits.yaml @@ -4075,6 +4075,327 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )e2e-v2-aws-backuprestore,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build09 + context: ci/prow/e2e-v2-aws-backuprestore-etcd-sharding + decorate: true + decoration_config: + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile + - Dockerfile.control-plane + - Dockerfile.e2e + labels: + capability/arm64: arm64 + ci-operator.openshift.io/cloud: hypershift-aws + ci-operator.openshift.io/cloud-cluster-profile: hypershift-aws + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-hypershift-main-e2e-v2-aws-backuprestore-etcd-sharding + optional: true + rerun_command: /test e2e-v2-aws-backuprestore-etcd-sharding + spec: + containers: + - args: + - --enable-secrets-store-csi-driver=true + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --gsm-config=/etc/gsm-config/gsm-config.yaml + - --gsm-credentials-file=/etc/gsm-credentials/key.json + - --gsm-project-config=/etc/gsm-config/gsm-project-config.yaml + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=e2e-v2-aws-backuprestore-etcd-sharding + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/gsm-config + name: gsm-config + readOnly: true + - mountPath: /etc/gsm-credentials + name: gsm-sa-key + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - configMap: + name: gsm-config + name: gsm-config + - csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: ci-operator-sa-key-spc + name: gsm-sa-key + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )e2e-v2-aws-backuprestore-etcd-sharding,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build09 + context: ci/prow/e2e-v2-aws-etcd-sharding + decorate: true + decoration_config: + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile + - Dockerfile.control-plane + - Dockerfile.e2e + labels: + capability/arm64: arm64 + ci-operator.openshift.io/cloud: hypershift-aws + ci-operator.openshift.io/cloud-cluster-profile: hypershift-aws + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-hypershift-main-e2e-v2-aws-etcd-sharding + optional: true + rerun_command: /test e2e-v2-aws-etcd-sharding + spec: + containers: + - args: + - --enable-secrets-store-csi-driver=true + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --gsm-config=/etc/gsm-config/gsm-config.yaml + - --gsm-credentials-file=/etc/gsm-credentials/key.json + - --gsm-project-config=/etc/gsm-config/gsm-project-config.yaml + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=e2e-v2-aws-etcd-sharding + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/gsm-config + name: gsm-config + readOnly: true + - mountPath: /etc/gsm-credentials + name: gsm-sa-key + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - configMap: + name: gsm-config + name: gsm-config + - csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: ci-operator-sa-key-spc + name: gsm-sa-key + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )e2e-v2-aws-etcd-sharding,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build09 + context: ci/prow/e2e-v2-aws-techpreview-osimagestream + decorate: true + decoration_config: + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile + - Dockerfile.control-plane + - Dockerfile.e2e + labels: + capability/arm64: arm64 + ci-operator.openshift.io/cloud: hypershift-aws + ci-operator.openshift.io/cloud-cluster-profile: hypershift-aws + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-hypershift-main-e2e-v2-aws-techpreview-osimagestream + optional: true + rerun_command: /test e2e-v2-aws-techpreview-osimagestream + spec: + containers: + - args: + - --enable-secrets-store-csi-driver=true + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --gsm-config=/etc/gsm-config/gsm-config.yaml + - --gsm-credentials-file=/etc/gsm-credentials/key.json + - --gsm-project-config=/etc/gsm-config/gsm-project-config.yaml + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=e2e-v2-aws-techpreview-osimagestream + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/gsm-config + name: gsm-config + readOnly: true + - mountPath: /etc/gsm-credentials + name: gsm-sa-key + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - configMap: + name: gsm-config + name: gsm-config + - csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: ci-operator-sa-key-spc + name: gsm-sa-key + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )e2e-v2-aws-techpreview-osimagestream,?($|\s.*) - agent: kubernetes always_run: false annotations: @@ -4183,6 +4504,113 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )e2e-v2-azure-self-managed,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build09 + context: ci/prow/e2e-v2-azure-self-managed-etcd-sharding + decorate: true + decoration_config: + sparse_checkout_files: + - .ci-operator.yaml + - Dockerfile + - Dockerfile.control-plane + - Dockerfile.e2e + labels: + capability/arm64: arm64 + ci-operator.openshift.io/cloud: hypershift-azure + ci-operator.openshift.io/cloud-cluster-profile: hypershift-azure + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-hypershift-main-e2e-v2-azure-self-managed-etcd-sharding + optional: true + rerun_command: /test e2e-v2-azure-self-managed-etcd-sharding + spec: + containers: + - args: + - --enable-secrets-store-csi-driver=true + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --gsm-config=/etc/gsm-config/gsm-config.yaml + - --gsm-credentials-file=/etc/gsm-credentials/key.json + - --gsm-project-config=/etc/gsm-config/gsm-project-config.yaml + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=e2e-v2-azure-self-managed-etcd-sharding + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/gsm-config + name: gsm-config + readOnly: true + - mountPath: /etc/gsm-credentials + name: gsm-sa-key + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - configMap: + name: gsm-config + name: gsm-config + - csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: ci-operator-sa-key-spc + name: gsm-sa-key + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )e2e-v2-azure-self-managed-etcd-sharding,?($|\s.*) - agent: kubernetes always_run: false annotations: diff --git a/ci-operator/step-registry/hypershift/aws/create-guests/hypershift-aws-create-guests-commands.sh b/ci-operator/step-registry/hypershift/aws/create-guests/hypershift-aws-create-guests-commands.sh index 52ecf9d24c17..ce160e4c7c65 100644 --- a/ci-operator/step-registry/hypershift/aws/create-guests/hypershift-aws-create-guests-commands.sh +++ b/ci-operator/step-registry/hypershift/aws/create-guests/hypershift-aws-create-guests-commands.sh @@ -11,7 +11,11 @@ if [[ -f "${SHARED_DIR}/nodepool_release_images" ]]; then fi if [[ -f "${SHARED_DIR}/test-plan.yaml" ]]; then + # An inline plan written by hypershift-write-test-plan wins over a plan + # that ships in the image, so a job can always override. export TEST_PLAN="${SHARED_DIR}/test-plan.yaml" +elif [[ -n "${TEST_PLAN_FILE:-}" ]]; then + export TEST_PLAN="${TEST_PLAN_FILE}" fi # Storage KMS encryption: only forward the KMS key alias when the hypershift diff --git a/ci-operator/step-registry/hypershift/aws/create-guests/hypershift-aws-create-guests-ref.yaml b/ci-operator/step-registry/hypershift/aws/create-guests/hypershift-aws-create-guests-ref.yaml index 931a6e56f203..2d4b66fe776e 100644 --- a/ci-operator/step-registry/hypershift/aws/create-guests/hypershift-aws-create-guests-ref.yaml +++ b/ci-operator/step-registry/hypershift/aws/create-guests/hypershift-aws-create-guests-ref.yaml @@ -39,6 +39,9 @@ ref: - name: HYPERSHIFT_STORAGE_KMS_KEY_ALIAS default: "" documentation: "AWS KMS key alias (e.g. alias/hypershift-ci) used to encrypt the default StorageClass volumes on the public cluster. When set and supported by the hypershift binary, the alias is resolved to an ARN and passed as --initial-storage-volumes-kms-key at cluster creation. Leave empty to skip." + - name: TEST_PLAN_FILE + default: "" + documentation: "Path to a test plan that ships in the hypershift-tests image, e.g. /hypershift/testplans/aws-etcd-sharding.yaml. Ignored when hypershift-write-test-plan has written an inline plan to SHARED_DIR. If both are empty, the platform's default test plan is used." from: hypershift-tests grace_period: 5m0s resources: diff --git a/ci-operator/step-registry/hypershift/aws/create/hypershift-aws-create-chain.yaml b/ci-operator/step-registry/hypershift/aws/create/hypershift-aws-create-chain.yaml index aa3dbdff06de..45bbe98708e8 100644 --- a/ci-operator/step-registry/hypershift/aws/create/hypershift-aws-create-chain.yaml +++ b/ci-operator/step-registry/hypershift/aws/create/hypershift-aws-create-chain.yaml @@ -33,6 +33,14 @@ chain: - name: EXTRA_ARGS default: "" documentation: "Extra args to pass to the create cluster aws command" + - name: HYPERSHIFT_ETCD_SHARDS + default: "" + documentation: | + Newline separated --etcd-shard values, one per managed etcd shard. Each value has the form + "name=,resources=/[;/...][,replicas=1|3][,storage=PersistentVolume|EmptyDir][,storageClassName=]"; + use an empty apiGroup for the core group. Requires the EtcdSharding feature gate, so the + HyperShift operator must be installed with TECH_PREVIEW_NO_UPGRADE=true, and shards with + replicas=3 require HYPERSHIFT_CP_AVAILABILITY_POLICY=HighlyAvailable. - name: HYPERSHIFT_GUEST_INFRA_OCP_ACCOUNT default: "false" documentation: "Whether to use the generic CI account or the HyperShift OSD account for the guest clusters infra. For the infra created for the clusters. E.g. For cluster-bot we use the generic CI account" @@ -174,6 +182,15 @@ chain: COMMAND+=(--network-type="${HYPERSHIFT_NETWORK_TYPE}") fi + # Shard values contain ";" as the resource separator, so they get their + # own quoted knob rather than riding along in the word-split EXTRA_ARGS. + if [[ -n $HYPERSHIFT_ETCD_SHARDS ]]; then + while IFS= read -r shard; do + [[ -n "$shard" ]] || continue + COMMAND+=(--etcd-shard="$shard") + done <<< "$HYPERSHIFT_ETCD_SHARDS" + fi + if [[ -n $EXTRA_ARGS ]]; then COMMAND+=(${EXTRA_ARGS}) fi diff --git a/ci-operator/step-registry/hypershift/aws/run-e2e-v2/hypershift-aws-run-e2e-v2-chain.yaml b/ci-operator/step-registry/hypershift/aws/run-e2e-v2/hypershift-aws-run-e2e-v2-chain.yaml index fb5f9b57520c..b8d11ab3f0df 100644 --- a/ci-operator/step-registry/hypershift/aws/run-e2e-v2/hypershift-aws-run-e2e-v2-chain.yaml +++ b/ci-operator/step-registry/hypershift/aws/run-e2e-v2/hypershift-aws-run-e2e-v2-chain.yaml @@ -7,6 +7,9 @@ chain: - name: HYPERSHIFT_PLATFORM default: "" documentation: "The cloud platform for the clusters (e.g. aws). Set in the workflow env block." + - name: TEST_PLAN_FILE + default: "" + documentation: "Path to a test plan that ships in the hypershift-tests image, e.g. /hypershift/testplans/aws-etcd-sharding.yaml. Must match the value given to hypershift-aws-create-guests so the same plan selects the clusters and the tests. Ignored when hypershift-write-test-plan has written an inline plan to SHARED_DIR." commands: |- set -xuo pipefail export KUBECONFIG="${SHARED_DIR}/management_cluster_kubeconfig" @@ -14,7 +17,11 @@ chain: export AWS_GUEST_INFRA_CREDENTIALS_FILE="/etc/hypershift-ci-jobs-awscreds/credentials" export PULL_SECRET_FILE="/etc/ci-pull-credentials/.dockerconfigjson" if [[ -f "${SHARED_DIR}/test-plan.yaml" ]]; then + # An inline plan written by hypershift-write-test-plan wins over a + # plan that ships in the image, so a job can always override. export TEST_PLAN="${SHARED_DIR}/test-plan.yaml" + elif [[ -n "${TEST_PLAN_FILE:-}" ]]; then + export TEST_PLAN="${TEST_PLAN_FILE}" fi /hypershift/bin/run-tests timeout: 150m diff --git a/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh b/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh index 0341fb58a9d5..047b635a4f7d 100644 --- a/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh +++ b/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-commands.sh @@ -10,4 +10,8 @@ if [[ -f "${SHARED_DIR}/nodepool_release_images" ]]; then source "${SHARED_DIR}/nodepool_release_images" fi +if [[ -n "${TEST_PLAN_FILE:-}" ]]; then + export TEST_PLAN="${TEST_PLAN_FILE}" +fi + /hypershift/bin/create-guests diff --git a/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-ref.yaml b/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-ref.yaml index 6fa8ef5a8b3d..3f6e8cb55566 100644 --- a/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-ref.yaml +++ b/ci-operator/step-registry/hypershift/azure/create-selfmanaged-guests/hypershift-azure-create-selfmanaged-guests-ref.yaml @@ -48,6 +48,9 @@ ref: - name: HYPERSHIFT_PLATFORM default: "" documentation: "The cloud platform for the self-managed clusters (e.g. azure). Set in the workflow env block." + - name: TEST_PLAN_FILE + default: "" + documentation: "Path to a test plan that ships in the hypershift-tests image, e.g. /hypershift/testplans/azure-etcd-sharding.yaml. If empty, the platform's default test plan is used." from: hypershift-tests grace_period: 5m0s resources: diff --git a/ci-operator/step-registry/hypershift/azure/run-e2e-v2-selfmanaged/hypershift-azure-run-e2e-v2-selfmanaged-chain.yaml b/ci-operator/step-registry/hypershift/azure/run-e2e-v2-selfmanaged/hypershift-azure-run-e2e-v2-selfmanaged-chain.yaml index c10066ef0dc3..2837395a7738 100644 --- a/ci-operator/step-registry/hypershift/azure/run-e2e-v2-selfmanaged/hypershift-azure-run-e2e-v2-selfmanaged-chain.yaml +++ b/ci-operator/step-registry/hypershift/azure/run-e2e-v2-selfmanaged/hypershift-azure-run-e2e-v2-selfmanaged-chain.yaml @@ -14,9 +14,15 @@ chain: - name: HYPERSHIFT_PLATFORM default: "" documentation: "The cloud platform for the self-managed clusters (e.g. azure). Set in the workflow env block." + - name: TEST_PLAN_FILE + default: "" + documentation: "Path to a test plan that ships in the hypershift-tests image, e.g. /hypershift/testplans/azure-etcd-sharding.yaml. Must match the value given to hypershift-azure-create-selfmanaged-guests so the same plan selects the clusters and the tests." commands: |- set -xuo pipefail export KUBECONFIG="${SHARED_DIR}/management_cluster_kubeconfig" + if [[ -n "${TEST_PLAN_FILE:-}" ]]; then + export TEST_PLAN="${TEST_PLAN_FILE}" + fi /hypershift/bin/run-tests timeout: 150m grace_period: 5m diff --git a/ci-operator/step-registry/hypershift/etcd-perf/OWNERS b/ci-operator/step-registry/hypershift/etcd-perf/OWNERS new file mode 100644 index 000000000000..588d9076e4d8 --- /dev/null +++ b/ci-operator/step-registry/hypershift/etcd-perf/OWNERS @@ -0,0 +1,15 @@ +approvers: +- bryan-cox +- cblecker +- csrwng +- enxebre +- jparrill +- mgencur +options: {} +reviewers: +- bryan-cox +- cblecker +- csrwng +- enxebre +- jparrill +- mgencur diff --git a/ci-operator/step-registry/hypershift/etcd-perf/README.md b/ci-operator/step-registry/hypershift/etcd-perf/README.md new file mode 100644 index 000000000000..66da920c89c2 --- /dev/null +++ b/ci-operator/step-registry/hypershift/etcd-perf/README.md @@ -0,0 +1,78 @@ +# hypershift-etcd-perf + +Runs one `kube-burner-ocp etcd-density` workload against a HyperShift hosted +cluster and measures the hosted control plane's etcd while it runs. + +This exists to give etcd sharding a periodic measurement, as opposed to the +`e2e-*-etcd-sharding` jobs, which are functional: they assert that shards get +deployed, that the kube-apiserver is given the right `--etcd-servers-overrides`, +that keys land in the shard they are routed to, and that the shards survive a +restart. Those jobs deliberately measure nothing. This one produces numbers and +nothing else. + +## Why a hosted cluster needs its own step + +On a standalone cluster, kube-burner points at one cluster and finds etcd in +`openshift-etcd` on that same cluster. On a hosted cluster the two halves are on +different clusters: the load lands on the guest API server, but the etcd under +test is a set of pods in `clusters-` on the management cluster. + +So this step: + +- drives the workload with `${SHARED_DIR}/nested_kubeconfig` (the guest), +- scrapes metrics from the management cluster's Thanos, via a metrics profile + re-scoped from `namespace="openshift-etcd"` to `namespace="clusters-"`, +- enables user workload monitoring on the management cluster first, because + without it nothing scrapes the hosted control plane's ServiceMonitors at all, +- verifies that etcd series are actually arriving before starting the workload, + so a job cannot burn hours producing load and no measurement. + +The `kube-burner-ocp-wrapper` in `cloud-bulldozer/e2e-benchmarking` does have a +hosted-control-plane mode, but its AWS path is ROSA specific: it derives the +control plane namespace from the `api.openshift.com/name` resource tag and it +expects an Observability Operator route on the management cluster. Neither holds +on the HyperShift CI lanes, which is why this step talks to `kube-burner-ocp` +directly. + +## Per-shard breakdown + +Every etcd series in the profile keeps its `pod` label. With sharding enabled +the pod names carry the shard name (`etcd-0`, `etcd-events-0`, `etcd-leases-0`, +...), so `by (pod)` is a per-shard breakdown for free. The shard topology in +effect is also dumped to `hostedcluster-etcd-spec.json` and `etcd-pods.txt` in +the artifact directory. + +## Comparing standard against sharded + +The step does not compare topologies. It measures whichever hosted cluster it is +pointed at and records `shardingVariant` in the result metadata. A comparison is +two jobs on the same schedule running the same workload with the same flags, +where one creates the hosted cluster with `HYPERSHIFT_ETCD_SHARDS` set and the +other does not. With `INDEX_TO_ES: "true"` both land in the perf and scale +Elasticsearch and Orion can do the diffing; without it, the numbers are in the +artifact directory under `collected-metrics-/`. + +## Workloads + +The four `etcd-density` sub-workloads are the ones the OpenShift Performance and +Scale team used to characterise standard against sharded on HCP: + +| Workload | What it stresses | +|---|---| +| `event-storm` | Bulk event creation against a concurrent victim workload | +| `crashloop-flood` | Continuous event pressure through the PATCH/count-increment path, which is the shape most real event storms take | +| `db-quota-pressure` | Large object fill driving the backend toward its storage quota | +| `annotation-churn` | Revision bloat from repeated annotation patching | + +Their flag sets differ; pass them through `ETCD_PERF_FLAGS` and check +`kube-burner-ocp etcd-density --help` for what each accepts. + +Note that `db-quota-pressure` needs `GC: "false"`, otherwise kube-burner removes +the objects that are supposed to be holding the backend under pressure. + +## Gating + +`ETCD_MAX_P99_COMMIT_LATENCY_MS` and `ETCD_MAX_LEADER_CHANGES` are both off by +default. When set they should be set generously: their job is to fail a run that +has gone obviously wrong, not to detect regressions. Regression detection is +Orion's, over many runs. diff --git a/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-commands.sh b/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-commands.sh new file mode 100644 index 000000000000..bcd189a50380 --- /dev/null +++ b/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-commands.sh @@ -0,0 +1,450 @@ +#!/bin/bash + +set -euo pipefail +# Do not enable `set -x`. This script mints a Prometheus bearer token and reads +# Elasticsearch credentials; tracing would put both in the CI logs. + +MC_KUBECONFIG="${SHARED_DIR}/management_cluster_kubeconfig" +GUEST_KUBECONFIG="${SHARED_DIR}/nested_kubeconfig" + +for f in "${MC_KUBECONFIG}" "${GUEST_KUBECONFIG}" "${SHARED_DIR}/cluster-name"; do + if [[ ! -s "${f}" ]]; then + echo "FATAL: ${f} is missing or empty; this step needs a hosted cluster to already exist." + exit 1 + fi +done + +CLUSTER_NAME="$(cat "${SHARED_DIR}/cluster-name")" +# HyperShift puts the hosted control plane in "-", +# and every CI workflow that feeds this step creates the HostedCluster in "clusters". +HCP_NAMESPACE="clusters-${CLUSTER_NAME}" + +mc() { oc --kubeconfig="${MC_KUBECONFIG}" "$@"; } + +echo "Hosted cluster: ${CLUSTER_NAME}" +echo "Hosted control plane ns: ${HCP_NAMESPACE}" +echo "Workload: etcd-density ${ETCD_PERF_WORKLOAD}" +echo "Shard topology variant: ${ETCD_PERF_VARIANT}" + +if ! mc get namespace "${HCP_NAMESPACE}" >/dev/null 2>&1; then + echo "FATAL: namespace ${HCP_NAMESPACE} does not exist on the management cluster." + exit 1 +fi + +echo +echo "=== etcd shards in ${HCP_NAMESPACE} ===" +# Recorded so the artifacts say which topology produced them. With sharding the +# pod names carry the shard name (etcd-0, etcd-events-0, ...), which is what +# makes the "by (pod)" breakdown in the metrics profile a per-shard breakdown. +mc get statefulset -n "${HCP_NAMESPACE}" -l app=etcd -o wide 2>/dev/null || true +mc get pods -n "${HCP_NAMESPACE}" -l app=etcd -o name 2>/dev/null | tee "${ARTIFACT_DIR}/etcd-pods.txt" || true +mc get hostedcluster -n clusters "${CLUSTER_NAME}" -o jsonpath='{.spec.etcd}' \ + > "${ARTIFACT_DIR}/hostedcluster-etcd-spec.json" 2>/dev/null || true +echo + +# --------------------------------------------------------------------------- +# Management cluster monitoring +# +# The hosted control plane's etcd and kube-apiserver run as ordinary workloads +# in ${HCP_NAMESPACE}, so their ServiceMonitors are only scraped if user +# workload monitoring is on. The management cluster here is ephemeral and +# per-job, so turning it on is safe, but still only do it if it is off. +# --------------------------------------------------------------------------- +enable_user_workload_monitoring() { + local existing + if ! existing="$(mc -n openshift-monitoring get configmap cluster-monitoring-config \ + -o jsonpath='{.data.config\.yaml}' 2>/dev/null)"; then + echo "Creating cluster-monitoring-config with user workload monitoring enabled" + mc apply -f - <<'EOF' +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-monitoring-config + namespace: openshift-monitoring +data: + config.yaml: | + enableUserWorkload: true +EOF + return + fi + + if grep -Eq '^[[:space:]]*enableUserWorkload:[[:space:]]*true' <<< "${existing}"; then + echo "User workload monitoring is already enabled" + return + fi + + echo "Appending enableUserWorkload to the existing cluster-monitoring-config" + printf '%s\nenableUserWorkload: true\n' "${existing}" > /tmp/cluster-monitoring-config.yaml + mc -n openshift-monitoring create configmap cluster-monitoring-config \ + --from-file=config.yaml=/tmp/cluster-monitoring-config.yaml \ + --dry-run=client -o yaml | mc replace -f - +} + +enable_user_workload_monitoring + +echo "Waiting for the user workload Prometheus to come up" +for _ in $(seq 1 60); do + if mc -n openshift-user-workload-monitoring get statefulset prometheus-user-workload >/dev/null 2>&1; then + break + fi + sleep 10 +done +mc -n openshift-user-workload-monitoring rollout status statefulset/prometheus-user-workload --timeout=10m + +# --------------------------------------------------------------------------- +# Thanos endpoint and token +# --------------------------------------------------------------------------- +PROM_SA="hypershift-etcd-perf" +mc -n openshift-monitoring create serviceaccount "${PROM_SA}" --dry-run=client -o yaml | mc apply -f - +mc adm policy add-cluster-role-to-user cluster-monitoring-view \ + -z "${PROM_SA}" -n openshift-monitoring >/dev/null + +THANOS_HOST="$(mc -n openshift-monitoring get route thanos-querier -o jsonpath='{.spec.host}')" +if [[ -z "${THANOS_HOST}" ]]; then + echo "FATAL: could not resolve the thanos-querier route on the management cluster." + exit 1 +fi +THANOS_URL="https://${THANOS_HOST}" +PROM_TOKEN="$(mc -n openshift-monitoring create token "${PROM_SA}" --duration=8h)" +echo "Thanos endpoint: ${THANOS_URL}" + +# The management cluster's ingress certificate is not one this container trusts, +# hence -k here and skipTLSVerify in the metrics endpoint below. +promq() { + curl -sSk --get --data-urlencode "query=$1" \ + -H "Authorization: Bearer ${PROM_TOKEN}" \ + "${THANOS_URL}/api/v1/query" +} + +echo "Waiting ${SCRAPE_SETTLE_SECONDS}s for the hosted control plane targets to be scraped" +sleep "${SCRAPE_SETTLE_SECONDS}" + +series_count="$(promq "count(etcd_server_has_leader{namespace=\"${HCP_NAMESPACE}\"})" \ + | jq -r '.data.result[0].value[1] // "0"')" +if [[ "${series_count}" == "0" ]]; then + echo "FATAL: Thanos returns no etcd metrics for namespace ${HCP_NAMESPACE}." + echo "Without them this run would produce a workload but no measurement, so fail now" + echo "rather than burn the cluster time. Check that the hosted control plane's etcd" + echo "ServiceMonitors exist and that user workload monitoring came up." + mc get servicemonitor -n "${HCP_NAMESPACE}" 2>&1 | head -20 || true + exit 1 +fi +echo "Scraping ${series_count} etcd member(s) in ${HCP_NAMESPACE}" + +# --------------------------------------------------------------------------- +# Metrics profile +# +# This is the upstream kube-burner-ocp etcd-density profile re-scoped for a +# hosted control plane: every series is confined to ${HCP_NAMESPACE} on the +# management cluster instead of openshift-etcd, and the etcd series keep their +# pod label so each shard is reported on its own. The metric set is the one the +# perf and scale team characterised standard vs sharded with. +# --------------------------------------------------------------------------- +PROFILE=/tmp/hcp-etcd-density-metrics.yml +cat > "${PROFILE}" < 0 + metricName: etcdCompaction +- query: etcd_debugging_mvcc_db_compaction_total_duration_milliseconds_sum{namespace="${HCP_NAMESPACE}"} + metricName: etcdCompactionDurationTotal +- query: delta(etcd_disk_backend_defrag_duration_seconds_sum{namespace="${HCP_NAMESPACE}"}[1m:30s])/2 > 0 + metricName: etcdDefrag +- query: sum by (pod) (rate(etcd_debugging_snap_save_total_duration_seconds_sum{namespace="${HCP_NAMESPACE}"}[2m])) + metricName: etcdSnapshotDuration + +# Raft +- query: sum by (pod) (rate(etcd_server_proposals_committed_total{namespace="${HCP_NAMESPACE}"}[2m])) + metricName: etcdProposalsCommittedRate +- query: sum by (pod) (rate(etcd_server_proposals_applied_total{namespace="${HCP_NAMESPACE}"}[2m])) + metricName: etcdProposalsAppliedRate +- query: sum by (pod) (rate(etcd_server_proposals_failed_total{namespace="${HCP_NAMESPACE}"}[2m])) + metricName: etcdProposalsFailedRate +- query: sum by (pod) (etcd_server_proposals_pending{namespace="${HCP_NAMESPACE}"}) + metricName: etcdProposalsPending +- query: sum by (pod) (rate(etcd_server_leader_changes_seen_total{namespace="${HCP_NAMESPACE}"}[2m])) + metricName: etcdLeaderChangesRate +- query: max by (pod) (etcd_server_has_leader{namespace="${HCP_NAMESPACE}"}) + metricName: etcdHasLeader + +# Slow operations +- query: delta(etcd_server_slow_apply_total{namespace="${HCP_NAMESPACE}"}[2m]) + metricName: etcdSlowApply +- query: delta(etcd_server_slow_read_indexes_total{namespace="${HCP_NAMESPACE}"}[2m]) + metricName: etcdSlowReadIndexes + +# Key space. With sharding this splits across pods, which is the whole point. +- query: etcd_debugging_mvcc_keys_total{namespace="${HCP_NAMESPACE}"} + metricName: etcdKeys +- query: etcd_debugging_mvcc_db_compaction_keys_total{namespace="${HCP_NAMESPACE}"} + metricName: etcdCompactedKeys +- query: rate(etcd_mvcc_put_total{namespace="${HCP_NAMESPACE}"}[2m]) + metricName: etcdPutRate +- query: rate(etcd_mvcc_delete_total{namespace="${HCP_NAMESPACE}"}[2m]) + metricName: etcdDeleteRate + +# Container resources. Sharding buys latency at the cost of running more etcd +# members, and the measured memory overhead was large, so track it per pod. +- query: (sum(irate(container_cpu_usage_seconds_total{namespace="${HCP_NAMESPACE}", container="etcd"}[2m]) * 100) by (pod)) > 0 + metricName: containerCPU +- query: sum(container_memory_working_set_bytes{namespace="${HCP_NAMESPACE}", container="etcd"}) by (pod) + metricName: containerMemory +- query: sum(container_memory_working_set_bytes{namespace="${HCP_NAMESPACE}"}) by (pod, container) + metricName: hcpContainerMemory + +# Management cluster node pressure. Not filtered to the nodes hosting this +# control plane: the node list is written to the artifact directory separately +# so the series can be narrowed during analysis. +- query: rate(node_vmstat_pgmajfault[2m]) + metricName: nodeMajorPageFaults + +# Hosted kube-apiserver. This is what the guest cluster actually feels when +# etcd slows down, and it is the number a shard routing change should move. +- query: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{namespace="${HCP_NAMESPACE}", subresource!="log", verb!~"WATCH|WATCHLIST|PROXY"}[2m])) by (verb, le)) + metricName: requestDuration99thByVerb +- query: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{namespace="${HCP_NAMESPACE}", subresource!="log", verb!~"WATCH|WATCHLIST|PROXY"}[2m])) by (resource, le)) + metricName: requestDuration99thByResource +- query: sum(rate(apiserver_request_total{namespace="${HCP_NAMESPACE}", verb!="WATCH"}[2m])) by (verb, resource, code) > 0 + metricName: APIRequestRate +- query: sum(apiserver_current_inflight_requests{namespace="${HCP_NAMESPACE}"}) by (request_kind) + metricName: requestInFlight +- query: sum(rate(apiserver_request_terminations_total{namespace="${HCP_NAMESPACE}"}[2m])) by (verb) + metricName: requestRateDropped +- query: topk(20, max by (resource) (apiserver_storage_objects{namespace="${HCP_NAMESPACE}"})) + metricName: etcdStorageObjects + +# Start and end snapshots +- query: etcd_mvcc_db_total_size_in_bytes{namespace="${HCP_NAMESPACE}"} + metricName: etcdDBTotalSizeSnapshot + instant: true + captureStart: true +- query: etcd_debugging_mvcc_keys_total{namespace="${HCP_NAMESPACE}"} + metricName: etcdKeysSnapshot + instant: true + captureStart: true +- query: etcd_server_proposals_committed_total{namespace="${HCP_NAMESPACE}"} + metricName: etcdProposalsCommitted + instant: true + captureStart: true +- query: etcd_server_slow_apply_total{namespace="${HCP_NAMESPACE}"} + metricName: etcdSlowApplyTotal + instant: true + captureStart: true +- query: etcd_server_leader_changes_seen_total{namespace="${HCP_NAMESPACE}"} + metricName: etcdLeaderChanges + instant: true + captureStart: true +- query: etcd_debugging_mvcc_db_compaction_total_duration_milliseconds_sum{namespace="${HCP_NAMESPACE}"} + metricName: etcdCompaction-raw + instant: true + captureStart: true +- query: sum(container_memory_working_set_bytes{namespace="${HCP_NAMESPACE}", container="etcd"}) by (pod) + metricName: containerMemorySnapshot + instant: true + captureStart: true +EOF +cp "${PROFILE}" "${ARTIFACT_DIR}/hcp-etcd-density-metrics.yml" + +# Nodes of the management cluster that are actually hosting this control plane, +# for narrowing the unfiltered node level series during analysis. +mc get pods -n "${HCP_NAMESPACE}" -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' \ + | sort -u | grep -v '^$' > "${ARTIFACT_DIR}/hcp-management-nodes.txt" || true + +# --------------------------------------------------------------------------- +# Metrics endpoints +# +# An endpoint with no indexer collects nothing that survives the run, so there +# is always a "local" one writing files that end up in the artifact directory. +# Indexing to Elasticsearch, when asked for, is a second endpoint over the same +# Thanos rather than a replacement, so a run is always inspectable by hand even +# when Orion is also getting the data. +# +# This file lives in /tmp and is never copied to ARTIFACT_DIR: it embeds the +# bearer token and, when indexing is on, the Elasticsearch credentials. +# --------------------------------------------------------------------------- +UUID="$(uuidgen)" +METRICS_DIR="/tmp/collected-metrics-${UUID}" +ENDPOINT_FILE=/tmp/metrics-endpoint.yml +{ + echo "- endpoint: ${THANOS_URL}" + echo " token: ${PROM_TOKEN}" + echo " skipTLSVerify: true" + echo " step: ${METRICS_STEP}" + echo " metrics:" + echo " - ${PROFILE}" + echo " indexer:" + echo " type: local" + echo " metricsDirectory: ${METRICS_DIR}" +} > "${ENDPOINT_FILE}" + +if [[ "${INDEX_TO_ES}" == "true" ]]; then + ES_HOST="search-ocp-qe-perf-scale-test-elk-hcm7wtsqpxy7xogbu72bor4uve.us-east-1.es.amazonaws.com" + if [[ -e "${ES_SECRETS_PATH}/host" ]]; then + ES_HOST="$(cat "${ES_SECRETS_PATH}/host")" + fi + ES_USERNAME="$(cat "${ES_SECRETS_PATH}/username")" + ES_PASSWORD="$(cat "${ES_SECRETS_PATH}/password")" + { + echo "- endpoint: ${THANOS_URL}" + echo " token: ${PROM_TOKEN}" + echo " skipTLSVerify: true" + echo " step: ${METRICS_STEP}" + echo " metrics:" + echo " - ${PROFILE}" + echo " indexer:" + echo " type: opensearch" + echo " esServers: [\"https://${ES_USERNAME}:${ES_PASSWORD}@${ES_HOST}\"]" + echo " defaultIndex: ${ES_INDEX}" + echo " insecureSkipVerify: true" + } >> "${ENDPOINT_FILE}" + echo "Results will also be indexed to Elasticsearch under uuid ${UUID}" +else + echo "INDEX_TO_ES is false; results stay in the artifact directory only" +fi + +# Metadata that makes a result set identifiable after the fact. Consumed by +# kube-burner as extra document fields, and kept as an artifact either way. +cat > /tmp/user-metadata.yaml < b)}'; then + FAILURES+=("peak p99 etcd backend commit latency was ${peak_ms}ms, above the ${ETCD_MAX_P99_COMMIT_LATENCY_MS}ms limit") + fi +fi + +if [[ -n "${ETCD_MAX_LEADER_CHANGES}" ]]; then + changes="$(promq "max(increase(etcd_server_leader_changes_seen_total{namespace=\"${HCP_NAMESPACE}\"}[2h]))" \ + | jq -r '.data.result[0].value[1] // "0"')" + printf 'etcd leader changes during the run: %.0f (limit %s)\n' "${changes}" "${ETCD_MAX_LEADER_CHANGES}" + if awk -v a="${changes}" -v b="${ETCD_MAX_LEADER_CHANGES}" 'BEGIN{exit !(a > b)}'; then + FAILURES+=("etcd saw ${changes} leader changes, above the ${ETCD_MAX_LEADER_CHANGES} limit") + fi +fi + +TESTCASE_NAME="etcd-density ${ETCD_PERF_WORKLOAD} (${ETCD_PERF_VARIANT})" +JUNIT="${ARTIFACT_DIR}/junit_hypershift_etcd_perf.xml" +if [[ ${RUN_EXIT_CODE} -ne 0 ]]; then + FAILURES+=("kube-burner-ocp exited with ${RUN_EXIT_CODE}") +fi + +if [[ ${#FAILURES[@]} -eq 0 ]]; then + cat > "${JUNIT}" < + + + +EOF + exit 0 +fi + +{ + echo '' + echo '' + echo " " + echo ' ' + echo ' ' + echo '' +} > "${JUNIT}" + +printf 'FAILED: %s\n' "${FAILURES[@]}" +exit 1 diff --git a/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-ref.metadata.json b/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-ref.metadata.json new file mode 100644 index 000000000000..f538ae53ed47 --- /dev/null +++ b/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-ref.metadata.json @@ -0,0 +1,21 @@ +{ + "path": "hypershift/etcd-perf/hypershift-etcd-perf-ref.yaml", + "owners": { + "approvers": [ + "bryan-cox", + "cblecker", + "csrwng", + "enxebre", + "jparrill", + "mgencur" + ], + "reviewers": [ + "bryan-cox", + "cblecker", + "csrwng", + "enxebre", + "jparrill", + "mgencur" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-ref.yaml b/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-ref.yaml new file mode 100644 index 000000000000..61e087d7d8e7 --- /dev/null +++ b/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-ref.yaml @@ -0,0 +1,112 @@ +ref: + as: hypershift-etcd-perf + from_image: + namespace: ci + name: ocp-qe-perfscale-ci + tag: latest + cli: latest + env: + - name: ETCD_PERF_WORKLOAD + default: "event-storm" + documentation: |- + Which "kube-burner-ocp etcd-density" sub-workload to run. One of + event-storm, crashloop-flood, db-quota-pressure, annotation-churn. + These are the four workloads the OpenShift Performance and Scale team + used for the standard-vs-sharded etcd characterisation on HCP. + - name: ETCD_PERF_FLAGS + default: "" + documentation: |- + Workload specific flags appended verbatim to the kube-burner-ocp + invocation, e.g. "--iterations=900 --event-iterations=2000". + See "kube-burner-ocp etcd-density --help" for the flag set; + it differs per workload. + - name: ETCD_PERF_VARIANT + default: "standard" + documentation: |- + Free form label recorded alongside the results so the sharded run can be + told apart from the standard baseline run when the two are compared. + Conventionally "standard" or "sharded"; it has no effect on what runs. + - name: QPS + default: "20" + documentation: "Client side QPS for kube-burner. 20 matches the perf and scale baseline." + - name: BURST + default: "40" + documentation: "Client side burst for kube-burner. 40 matches the perf and scale baseline." + - name: GC + default: "true" + documentation: |- + Whether kube-burner deletes the objects it created. Set to false for + db-quota-pressure, which has to retain its objects to keep the backend + under storage pressure for the duration of the measurement. + - name: KUBE_BURNER_VERSION + default: "1.12.6" + documentation: "kube-burner-ocp release to download." + - name: KUBE_BURNER_URL + default: "" + documentation: "Full URL of a kube-burner-ocp tarball, overriding KUBE_BURNER_VERSION." + - name: METRICS_STEP + default: "30s" + documentation: "Prometheus range query resolution used when collecting the metrics profile." + - name: SCRAPE_SETTLE_SECONDS + default: "180" + documentation: |- + How long to wait after enabling user workload monitoring before checking + that the hosted control plane etcd targets are actually being scraped. + - name: INDEX_TO_ES + default: "false" + documentation: |- + When true, index the collected metrics into the perf and scale + Elasticsearch so Orion can do run-over-run regression detection. + When false the metrics are only written to the artifact directory. + - name: ES_SECRETS_PATH + default: "/secret" + documentation: "Directory holding the Elasticsearch username, password and optional host files." + - name: ES_INDEX + default: "ripsaw-kube-burner" + documentation: "Elasticsearch index to write the collected metrics to." + - name: ETCD_MAX_P99_COMMIT_LATENCY_MS + default: "" + documentation: |- + Optional hard gate. If set, the step fails when the peak 99th percentile + etcd backend commit latency observed during the run exceeds this many + milliseconds. Leave empty to collect data without gating; the intended + use is a deliberately generous bound that only catches gross regressions, + with trend detection left to Orion over the indexed results. + - name: ETCD_MAX_LEADER_CHANGES + default: "" + documentation: |- + Optional hard gate. If set, the step fails when any etcd shard saw more + than this many leader changes during the run. Leave empty to disable. + commands: hypershift-etcd-perf-commands.sh + credentials: + - collection: ocp-perfscale-cpt + group: ocp-qe-perfscale-es-creds + mount_path: /secret + resources: + requests: + cpu: 100m + memory: 200Mi + timeout: 4h0m0s + grace_period: 10m0s + documentation: |- + Runs one of the kube-burner-ocp "etcd-density" workloads against a + HyperShift hosted cluster and collects the etcd and kube-apiserver metrics + of that cluster's hosted control plane. + + The interesting property of a hosted cluster is that the load is applied to + the guest API server while the etcd being measured runs as pods in the + "clusters-" namespace of the management cluster. So this step drives + the workload with the guest kubeconfig and scrapes metrics from the + management cluster's Thanos, using a metrics profile scoped to the hosted + control plane namespace and broken down "by (pod)" so that each etcd shard + is reported separately. + + It does not compare shard topologies by itself. Comparison is done by + running the same workload twice, once against a hosted cluster created with + "--etcd-shard" flags and once without, and diffing the two result sets -- + either by hand from the artifact directory or, when INDEX_TO_ES is true, + with Orion over the indexed metrics. + + Requires a hosted cluster to already exist, i.e. the hypershift-aws-create + chain (or an equivalent) must have run and written ${SHARED_DIR}/cluster-name + and ${SHARED_DIR}/nested_kubeconfig. From e53150242379fb2e41d867bc29d6751442bd6583 Mon Sep 17 00:00:00 2001 From: mehabhalodiya Date: Mon, 26 Oct 2026 16:57:56 +0000 Subject: [PATCH 2/2] ci(hypershift): rework etcd perf jobs around reusable steps Replaces the single hypershift-etcd-perf step with a three step chain, after review feedback on openshift/release#86142 and the sharding review with the performance and scale team. The step carried its own copy of the kube-burner etcd-density metrics profile, built its own metrics endpoint, and assembled an authenticated Elasticsearch URL. None of that needed to be HyperShift's: - The metrics queries now come from metrics-profiles/ in cloud-bulldozer/e2e-benchmarking, at the tag the openshift-qe workload steps resolve, so there is one copy of them owned by the team that owns the measurements. - User workload monitoring is its own step. A hosted control plane is only one of the things that needs it; any operator shipping a ServiceMonitor in a namespace the monitoring stack does not own needs the same. - The workload is openshift-qe-etcd-density-annotation-churn, used unmodified, so the HCP numbers and the standalone OpenShift numbers come from the same workload definition. That step grows a self-managed HyperShift branch, which is a no-op anywhere the chain has not run. - No Elasticsearch credentials are constructed in HyperShift-owned code. Shard topology is the one agreed with the performance and scale team: three PersistentVolume backed main members, a three member PersistentVolume backed leases shard so cluster heartbeats get their own highly available shard, and a single EmptyDir backed events shard. Node count matches the 24 worker bed the characterisation ran on. Workload variants are added one per pull request, starting with annotation churn; crashloop-flood, db-quota-pressure and event-storm follow. Also stops eval'ing the create command in the NodePool skew path. An --etcd-shard value separates resources with ";", which eval would read as shell syntax rather than as one argument. Co-Authored-By: Claude Opus 5 --- ...-hypershift-main__periodics-etcd-perf.yaml | 74 +-- .../openshift-hypershift-main-periodics.yaml | 214 +-------- .../openshift-hypershift-main-presubmits.yaml | 107 ----- .../create/hypershift-aws-create-chain.yaml | 22 +- .../enable-user-workload-monitoring/OWNERS | 1 + ...nable-user-workload-monitoring-commands.sh | 56 +++ ...user-workload-monitoring-ref.metadata.json | 21 + ...t-enable-user-workload-monitoring-ref.yaml | 37 ++ .../step-registry/hypershift/etcd-perf/OWNERS | 16 +- .../hypershift/etcd-perf/README.md | 125 +++-- ... hypershift-etcd-perf-chain.metadata.json} | 2 +- .../etcd-perf/hypershift-etcd-perf-chain.yaml | 34 ++ .../hypershift-etcd-perf-commands.sh | 450 ------------------ .../etcd-perf/hypershift-etcd-perf-ref.yaml | 112 ----- .../hypershift/hcp-metrics-endpoint/OWNERS | 1 + ...ypershift-hcp-metrics-endpoint-commands.sh | 155 ++++++ ...ift-hcp-metrics-endpoint-ref.metadata.json | 21 + .../hypershift-hcp-metrics-endpoint-ref.yaml | 71 +++ ...-etcd-density-annotation-churn-commands.sh | 34 +- 19 files changed, 550 insertions(+), 1003 deletions(-) create mode 120000 ci-operator/step-registry/hypershift/enable-user-workload-monitoring/OWNERS create mode 100755 ci-operator/step-registry/hypershift/enable-user-workload-monitoring/hypershift-enable-user-workload-monitoring-commands.sh create mode 100644 ci-operator/step-registry/hypershift/enable-user-workload-monitoring/hypershift-enable-user-workload-monitoring-ref.metadata.json create mode 100644 ci-operator/step-registry/hypershift/enable-user-workload-monitoring/hypershift-enable-user-workload-monitoring-ref.yaml mode change 100644 => 120000 ci-operator/step-registry/hypershift/etcd-perf/OWNERS rename ci-operator/step-registry/hypershift/etcd-perf/{hypershift-etcd-perf-ref.metadata.json => hypershift-etcd-perf-chain.metadata.json} (78%) create mode 100644 ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-chain.yaml delete mode 100644 ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-commands.sh delete mode 100644 ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-ref.yaml create mode 120000 ci-operator/step-registry/hypershift/hcp-metrics-endpoint/OWNERS create mode 100755 ci-operator/step-registry/hypershift/hcp-metrics-endpoint/hypershift-hcp-metrics-endpoint-commands.sh create mode 100644 ci-operator/step-registry/hypershift/hcp-metrics-endpoint/hypershift-hcp-metrics-endpoint-ref.metadata.json create mode 100644 ci-operator/step-registry/hypershift/hcp-metrics-endpoint/hypershift-hcp-metrics-endpoint-ref.yaml diff --git a/ci-operator/config/openshift/hypershift/openshift-hypershift-main__periodics-etcd-perf.yaml b/ci-operator/config/openshift/hypershift/openshift-hypershift-main__periodics-etcd-perf.yaml index f4198111d6fe..59298f238957 100644 --- a/ci-operator/config/openshift/hypershift/openshift-hypershift-main__periodics-etcd-perf.yaml +++ b/ci-operator/config/openshift/hypershift/openshift-hypershift-main__periodics-etcd-perf.yaml @@ -30,82 +30,38 @@ resources: cpu: 100m memory: 200Mi tests: -- as: event-storm-standard +- as: annotation-churn-standard cron: 20 7 * * 2 steps: cluster_profile: hypershift-aws env: - ETCD_MAX_LEADER_CHANGES: "5" - ETCD_MAX_P99_COMMIT_LATENCY_MS: "500" - ETCD_PERF_FLAGS: --iterations=60 --pod-replicas=2 --event-iterations=2000 --event-replicas=100 - --storm-workers=20 - ETCD_PERF_VARIANT: standard - ETCD_PERF_WORKLOAD: event-storm + ANNOTATION_CHURN_ITERATIONS: "10" HYPERSHIFT_CP_AVAILABILITY_POLICY: HighlyAvailable HYPERSHIFT_INSTANCE_TYPE: m5.xlarge - HYPERSHIFT_NODE_COUNT: "6" + HYPERSHIFT_NODE_COUNT: "24" + PATCH_REPLICAS: "50" + PATCH_ROUNDS: "100" test: - - ref: hypershift-etcd-perf + - chain: hypershift-etcd-perf workflow: hypershift-aws-e2e-cluster timeout: 6h0m0s -- as: event-storm-sharded +- as: annotation-churn-sharded cron: 20 13 * * 2 steps: cluster_profile: hypershift-aws env: - ETCD_MAX_LEADER_CHANGES: "5" - ETCD_MAX_P99_COMMIT_LATENCY_MS: "500" - ETCD_PERF_FLAGS: --iterations=60 --pod-replicas=2 --event-iterations=2000 --event-replicas=100 - --storm-workers=20 - ETCD_PERF_VARIANT: sharded - ETCD_PERF_WORKLOAD: event-storm + ANNOTATION_CHURN_ITERATIONS: "10" HYPERSHIFT_CP_AVAILABILITY_POLICY: HighlyAvailable - HYPERSHIFT_ETCD_SHARDS: name=events,resources=/events;events.k8s.io/events,replicas=3,storage=PersistentVolume + HYPERSHIFT_ETCD_SHARDS: |- + name=leases,resources=coordination.k8s.io/leases,replicas=3,storage=PersistentVolume + name=events,resources=/events;events.k8s.io/events,replicas=1,storage=EmptyDir HYPERSHIFT_INSTANCE_TYPE: m5.xlarge - HYPERSHIFT_NODE_COUNT: "6" + HYPERSHIFT_NODE_COUNT: "24" + PATCH_REPLICAS: "50" + PATCH_ROUNDS: "100" TECH_PREVIEW_NO_UPGRADE: "true" test: - - ref: hypershift-etcd-perf - workflow: hypershift-aws-e2e-cluster - timeout: 6h0m0s -- as: db-quota-pressure-standard - cron: 20 7 * * 4 - steps: - cluster_profile: hypershift-aws - env: - ETCD_MAX_LEADER_CHANGES: "5" - ETCD_MAX_P99_COMMIT_LATENCY_MS: "500" - ETCD_PERF_FLAGS: --iterations=2000 --iterations-per-namespace=10 --kb-chunks=8 - --kb-size=100 - ETCD_PERF_VARIANT: standard - ETCD_PERF_WORKLOAD: db-quota-pressure - GC: "false" - HYPERSHIFT_CP_AVAILABILITY_POLICY: HighlyAvailable - HYPERSHIFT_INSTANCE_TYPE: m5.xlarge - HYPERSHIFT_NODE_COUNT: "3" - test: - - ref: hypershift-etcd-perf - workflow: hypershift-aws-e2e-cluster - timeout: 6h0m0s -- as: db-quota-pressure-sharded - cron: 20 13 * * 4 - steps: - cluster_profile: hypershift-aws - env: - ETCD_MAX_LEADER_CHANGES: "5" - ETCD_MAX_P99_COMMIT_LATENCY_MS: "500" - ETCD_PERF_FLAGS: --iterations=2000 --iterations-per-namespace=10 --kb-chunks=8 - --kb-size=100 - ETCD_PERF_VARIANT: sharded - ETCD_PERF_WORKLOAD: db-quota-pressure - GC: "false" - HYPERSHIFT_CP_AVAILABILITY_POLICY: HighlyAvailable - HYPERSHIFT_ETCD_SHARDS: name=events,resources=/events;events.k8s.io/events,replicas=3,storage=PersistentVolume - HYPERSHIFT_INSTANCE_TYPE: m5.xlarge - HYPERSHIFT_NODE_COUNT: "3" - TECH_PREVIEW_NO_UPGRADE: "true" - test: - - ref: hypershift-etcd-perf + - chain: hypershift-etcd-perf workflow: hypershift-aws-e2e-cluster timeout: 6h0m0s zz_generated_metadata: diff --git a/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-periodics.yaml b/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-periodics.yaml index dc8253b864f3..61111be1b902 100644 --- a/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-periodics.yaml +++ b/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-periodics.yaml @@ -820,212 +820,6 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator -- agent: kubernetes - cluster: build05 - cron: 20 13 * * 4 - decorate: true - decoration_config: - skip_cloning: true - timeout: 6h0m0s - extra_refs: - - base_ref: main - org: openshift - repo: hypershift - labels: - ci-operator.openshift.io/cloud: hypershift-aws - ci-operator.openshift.io/cloud-cluster-profile: hypershift-aws - ci-operator.openshift.io/variant: periodics-etcd-perf - ci.openshift.io/generator: prowgen - job-release: "5.1" - pj-rehearse.openshift.io/can-be-rehearsed: "true" - name: periodic-ci-openshift-hypershift-main-periodics-etcd-perf-db-quota-pressure-sharded - spec: - containers: - - args: - - --enable-secrets-store-csi-driver=true - - --gcs-upload-secret=/secrets/gcs/service-account.json - - --gsm-config=/etc/gsm-config/gsm-config.yaml - - --gsm-credentials-file=/etc/gsm-credentials/key.json - - --gsm-project-config=/etc/gsm-config/gsm-project-config.yaml - - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson - - --lease-server-credentials-file=/etc/boskos/credentials - - --report-credentials-file=/etc/report/credentials - - --secret-dir=/secrets/ci-pull-credentials - - --target=db-quota-pressure-sharded - - --variant=periodics-etcd-perf - command: - - ci-operator - env: - - name: HTTP_SERVER_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest - imagePullPolicy: Always - name: "" - ports: - - containerPort: 8080 - name: http - resources: - requests: - cpu: 10m - volumeMounts: - - mountPath: /etc/boskos - name: boskos - readOnly: true - - mountPath: /secrets/ci-pull-credentials - name: ci-pull-credentials - readOnly: true - - mountPath: /secrets/gcs - name: gcs-credentials - readOnly: true - - mountPath: /etc/gsm-config - name: gsm-config - readOnly: true - - mountPath: /etc/gsm-credentials - name: gsm-sa-key - readOnly: true - - mountPath: /secrets/manifest-tool - name: manifest-tool-local-pusher - readOnly: true - - mountPath: /etc/pull-secret - name: pull-secret - readOnly: true - - mountPath: /etc/report - name: result-aggregator - readOnly: true - serviceAccountName: ci-operator - volumes: - - name: boskos - secret: - items: - - key: credentials - path: credentials - secretName: boskos-credentials - - name: ci-pull-credentials - secret: - secretName: ci-pull-credentials - - configMap: - name: gsm-config - name: gsm-config - - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: ci-operator-sa-key-spc - name: gsm-sa-key - - name: manifest-tool-local-pusher - secret: - secretName: manifest-tool-local-pusher - - name: pull-secret - secret: - secretName: registry-pull-credentials - - name: result-aggregator - secret: - secretName: result-aggregator -- agent: kubernetes - cluster: build05 - cron: 20 7 * * 4 - decorate: true - decoration_config: - skip_cloning: true - timeout: 6h0m0s - extra_refs: - - base_ref: main - org: openshift - repo: hypershift - labels: - ci-operator.openshift.io/cloud: hypershift-aws - ci-operator.openshift.io/cloud-cluster-profile: hypershift-aws - ci-operator.openshift.io/variant: periodics-etcd-perf - ci.openshift.io/generator: prowgen - job-release: "5.1" - pj-rehearse.openshift.io/can-be-rehearsed: "true" - name: periodic-ci-openshift-hypershift-main-periodics-etcd-perf-db-quota-pressure-standard - spec: - containers: - - args: - - --enable-secrets-store-csi-driver=true - - --gcs-upload-secret=/secrets/gcs/service-account.json - - --gsm-config=/etc/gsm-config/gsm-config.yaml - - --gsm-credentials-file=/etc/gsm-credentials/key.json - - --gsm-project-config=/etc/gsm-config/gsm-project-config.yaml - - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson - - --lease-server-credentials-file=/etc/boskos/credentials - - --report-credentials-file=/etc/report/credentials - - --secret-dir=/secrets/ci-pull-credentials - - --target=db-quota-pressure-standard - - --variant=periodics-etcd-perf - command: - - ci-operator - env: - - name: HTTP_SERVER_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest - imagePullPolicy: Always - name: "" - ports: - - containerPort: 8080 - name: http - resources: - requests: - cpu: 10m - volumeMounts: - - mountPath: /etc/boskos - name: boskos - readOnly: true - - mountPath: /secrets/ci-pull-credentials - name: ci-pull-credentials - readOnly: true - - mountPath: /secrets/gcs - name: gcs-credentials - readOnly: true - - mountPath: /etc/gsm-config - name: gsm-config - readOnly: true - - mountPath: /etc/gsm-credentials - name: gsm-sa-key - readOnly: true - - mountPath: /secrets/manifest-tool - name: manifest-tool-local-pusher - readOnly: true - - mountPath: /etc/pull-secret - name: pull-secret - readOnly: true - - mountPath: /etc/report - name: result-aggregator - readOnly: true - serviceAccountName: ci-operator - volumes: - - name: boskos - secret: - items: - - key: credentials - path: credentials - secretName: boskos-credentials - - name: ci-pull-credentials - secret: - secretName: ci-pull-credentials - - configMap: - name: gsm-config - name: gsm-config - - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: ci-operator-sa-key-spc - name: gsm-sa-key - - name: manifest-tool-local-pusher - secret: - secretName: manifest-tool-local-pusher - - name: pull-secret - secret: - secretName: registry-pull-credentials - - name: result-aggregator - secret: - secretName: result-aggregator - agent: kubernetes cluster: build05 cron: 20 13 * * 2 @@ -1044,7 +838,7 @@ periodics: ci.openshift.io/generator: prowgen job-release: "5.1" pj-rehearse.openshift.io/can-be-rehearsed: "true" - name: periodic-ci-openshift-hypershift-main-periodics-etcd-perf-event-storm-sharded + name: periodic-ci-openshift-hypershift-main-periodics-etcd-perf-annotation-churn-sharded spec: containers: - args: @@ -1057,7 +851,7 @@ periodics: - --lease-server-credentials-file=/etc/boskos/credentials - --report-credentials-file=/etc/report/credentials - --secret-dir=/secrets/ci-pull-credentials - - --target=event-storm-sharded + - --target=annotation-churn-sharded - --variant=periodics-etcd-perf command: - ci-operator @@ -1147,7 +941,7 @@ periodics: ci.openshift.io/generator: prowgen job-release: "5.1" pj-rehearse.openshift.io/can-be-rehearsed: "true" - name: periodic-ci-openshift-hypershift-main-periodics-etcd-perf-event-storm-standard + name: periodic-ci-openshift-hypershift-main-periodics-etcd-perf-annotation-churn-standard spec: containers: - args: @@ -1160,7 +954,7 @@ periodics: - --lease-server-credentials-file=/etc/boskos/credentials - --report-credentials-file=/etc/report/credentials - --secret-dir=/secrets/ci-pull-credentials - - --target=event-storm-standard + - --target=annotation-churn-standard - --variant=periodics-etcd-perf command: - ci-operator diff --git a/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-presubmits.yaml b/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-presubmits.yaml index 39f6b9330f31..880a3bae2074 100644 --- a/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-presubmits.yaml +++ b/ci-operator/jobs/openshift/hypershift/openshift-hypershift-main-presubmits.yaml @@ -4289,113 +4289,6 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )e2e-v2-aws-etcd-sharding,?($|\s.*) - - agent: kubernetes - always_run: false - branches: - - ^main$ - - ^main- - cluster: build09 - context: ci/prow/e2e-v2-aws-techpreview-osimagestream - decorate: true - decoration_config: - sparse_checkout_files: - - .ci-operator.yaml - - Dockerfile - - Dockerfile.control-plane - - Dockerfile.e2e - labels: - capability/arm64: arm64 - ci-operator.openshift.io/cloud: hypershift-aws - ci-operator.openshift.io/cloud-cluster-profile: hypershift-aws - ci.openshift.io/generator: prowgen - pj-rehearse.openshift.io/can-be-rehearsed: "true" - name: pull-ci-openshift-hypershift-main-e2e-v2-aws-techpreview-osimagestream - optional: true - rerun_command: /test e2e-v2-aws-techpreview-osimagestream - spec: - containers: - - args: - - --enable-secrets-store-csi-driver=true - - --gcs-upload-secret=/secrets/gcs/service-account.json - - --gsm-config=/etc/gsm-config/gsm-config.yaml - - --gsm-credentials-file=/etc/gsm-credentials/key.json - - --gsm-project-config=/etc/gsm-config/gsm-project-config.yaml - - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson - - --lease-server-credentials-file=/etc/boskos/credentials - - --report-credentials-file=/etc/report/credentials - - --secret-dir=/secrets/ci-pull-credentials - - --target=e2e-v2-aws-techpreview-osimagestream - command: - - ci-operator - env: - - name: HTTP_SERVER_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest - imagePullPolicy: Always - name: "" - ports: - - containerPort: 8080 - name: http - resources: - requests: - cpu: 10m - volumeMounts: - - mountPath: /etc/boskos - name: boskos - readOnly: true - - mountPath: /secrets/ci-pull-credentials - name: ci-pull-credentials - readOnly: true - - mountPath: /secrets/gcs - name: gcs-credentials - readOnly: true - - mountPath: /etc/gsm-config - name: gsm-config - readOnly: true - - mountPath: /etc/gsm-credentials - name: gsm-sa-key - readOnly: true - - mountPath: /secrets/manifest-tool - name: manifest-tool-local-pusher - readOnly: true - - mountPath: /etc/pull-secret - name: pull-secret - readOnly: true - - mountPath: /etc/report - name: result-aggregator - readOnly: true - serviceAccountName: ci-operator - volumes: - - name: boskos - secret: - items: - - key: credentials - path: credentials - secretName: boskos-credentials - - name: ci-pull-credentials - secret: - secretName: ci-pull-credentials - - configMap: - name: gsm-config - name: gsm-config - - csi: - driver: secrets-store.csi.k8s.io - readOnly: true - volumeAttributes: - secretProviderClass: ci-operator-sa-key-spc - name: gsm-sa-key - - name: manifest-tool-local-pusher - secret: - secretName: manifest-tool-local-pusher - - name: pull-secret - secret: - secretName: registry-pull-credentials - - name: result-aggregator - secret: - secretName: result-aggregator - trigger: (?m)^/test( | .* )e2e-v2-aws-techpreview-osimagestream,?($|\s.*) - agent: kubernetes always_run: false annotations: diff --git a/ci-operator/step-registry/hypershift/aws/create/hypershift-aws-create-chain.yaml b/ci-operator/step-registry/hypershift/aws/create/hypershift-aws-create-chain.yaml index 45bbe98708e8..26351ba84107 100644 --- a/ci-operator/step-registry/hypershift/aws/create/hypershift-aws-create-chain.yaml +++ b/ci-operator/step-registry/hypershift/aws/create/hypershift-aws-create-chain.yaml @@ -41,6 +41,13 @@ chain: use an empty apiGroup for the core group. Requires the EtcdSharding feature gate, so the HyperShift operator must be installed with TECH_PREVIEW_NO_UPGRADE=true, and shards with replicas=3 require HYPERSHIFT_CP_AVAILABILITY_POLICY=HighlyAvailable. + + The topology the performance jobs use, agreed with the performance and scale team, is a + three member PersistentVolume backed main etcd plus: + name=leases,resources=coordination.k8s.io/leases,replicas=3,storage=PersistentVolume + name=events,resources=/events;events.k8s.io/events,replicas=1,storage=EmptyDir + Leases get their own highly available shard because they carry the cluster heartbeat, and + events get a single EmptyDir shard because they are high churn and disposable. - name: HYPERSHIFT_GUEST_INFRA_OCP_ACCOUNT default: "false" documentation: "Whether to use the generic CI account or the HyperShift OSD account for the guest clusters infra. For the infra created for the clusters. E.g. For cluster-bot we use the generic CI account" @@ -113,13 +120,15 @@ chain: AWS_GUEST_INFRA_CREDENTIALS_FILE="/etc/hypershift-ci-jobs-awscreds/credentials" DEFAULT_BASE_DOMAIN=ci.hypershift.devcluster.openshift.com + # Renders the HostedCluster, rewrites the NodePool release image and applies + # the result. COMMAND is passed as an argument array rather than a shell + # string so that values containing shell metacharacters -- most notably the + # ";" that separates resources inside an --etcd-shard value -- survive. support_np_skew() { curl -L https://github.com/mikefarah/yq/releases/download/v4.31.2/yq_linux_amd64 -o /tmp/yq && chmod +x /tmp/yq - local extra_flags="" - extra_flags+="--render-sensitive --render > /tmp/hc.yaml " - extra_flags+="&& /tmp/yq e -i '(select(.kind == \"NodePool\").spec.release.image) = \"$NODEPOOL_RELEASE_IMAGE_LATEST\"' /tmp/hc.yaml " - extra_flags+="&& oc apply -f /tmp/hc.yaml" - echo "$extra_flags" + "${COMMAND[@]}" --render-sensitive --render > /tmp/hc.yaml + /tmp/yq e -i "(select(.kind == \"NodePool\").spec.release.image) = \"$NODEPOOL_RELEASE_IMAGE_LATEST\"" /tmp/hc.yaml + oc apply -f /tmp/hc.yaml } HC_REGION=${HYPERSHIFT_AWS_REGION:-$LEASED_RESOURCE} @@ -247,9 +256,8 @@ chain: fi # Execute the command - # When using np_skew (different release images), we need eval to properly handle shell operators if [[ -n "$RELEASE_IMAGE_LATEST" && -n "$NODEPOOL_RELEASE_IMAGE_LATEST" && "$RELEASE_IMAGE_LATEST" != "$NODEPOOL_RELEASE_IMAGE_LATEST" ]]; then - eval "${COMMAND[*]} $(support_np_skew)" + support_np_skew else "${COMMAND[@]}" fi diff --git a/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/OWNERS b/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/OWNERS new file mode 120000 index 000000000000..ec405d65a79d --- /dev/null +++ b/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/OWNERS @@ -0,0 +1 @@ +../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/hypershift-enable-user-workload-monitoring-commands.sh b/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/hypershift-enable-user-workload-monitoring-commands.sh new file mode 100755 index 000000000000..894ff15e69e5 --- /dev/null +++ b/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/hypershift-enable-user-workload-monitoring-commands.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +set -euo pipefail +# No `set -x`: this step does not handle credentials, but it is a sibling of +# steps that do and the surrounding convention is to keep tracing off. + +KUBECONFIG_FILE="${MONITORING_KUBECONFIG:-${SHARED_DIR}/management_cluster_kubeconfig}" +if [[ ! -s "${KUBECONFIG_FILE}" ]]; then + echo "FATAL: ${KUBECONFIG_FILE} is missing or empty." + exit 1 +fi + +k() { oc --kubeconfig="${KUBECONFIG_FILE}" "$@"; } + +# A hosted control plane runs as ordinary workloads in a user namespace of the +# management cluster, so its ServiceMonitors are only scraped when user workload +# monitoring is on. The same is true for any operator shipping a ServiceMonitor +# in a namespace the cluster monitoring stack does not own, which is why this is +# a step of its own rather than part of a single workload. +existing="" +if ! existing="$(k -n openshift-monitoring get configmap cluster-monitoring-config \ + -o jsonpath='{.data.config\.yaml}' 2>/dev/null)"; then + echo "Creating cluster-monitoring-config with user workload monitoring enabled" + k apply -f - <<'EOF' +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-monitoring-config + namespace: openshift-monitoring +data: + config.yaml: | + enableUserWorkload: true +EOF +elif grep -Eq '^[[:space:]]*enableUserWorkload:[[:space:]]*true' <<< "${existing}"; then + echo "User workload monitoring is already enabled" +else + echo "Appending enableUserWorkload to the existing cluster-monitoring-config" + printf '%s\nenableUserWorkload: true\n' "${existing}" > /tmp/cluster-monitoring-config.yaml + k -n openshift-monitoring create configmap cluster-monitoring-config \ + --from-file=config.yaml=/tmp/cluster-monitoring-config.yaml \ + --dry-run=client -o yaml | k replace -f - +fi + +echo "Waiting for the user workload Prometheus StatefulSet to be created" +for _ in $(seq 1 "${UWM_WAIT_ATTEMPTS}"); do + if k -n openshift-user-workload-monitoring get statefulset prometheus-user-workload >/dev/null 2>&1; then + break + fi + sleep 10 +done + +k -n openshift-user-workload-monitoring rollout status \ + statefulset/prometheus-user-workload --timeout="${UWM_ROLLOUT_TIMEOUT}" + +k -n openshift-user-workload-monitoring get pods \ + > "${ARTIFACT_DIR}/user-workload-monitoring-pods.txt" 2>&1 || true diff --git a/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/hypershift-enable-user-workload-monitoring-ref.metadata.json b/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/hypershift-enable-user-workload-monitoring-ref.metadata.json new file mode 100644 index 000000000000..d50d1d4c3dbb --- /dev/null +++ b/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/hypershift-enable-user-workload-monitoring-ref.metadata.json @@ -0,0 +1,21 @@ +{ + "path": "hypershift/enable-user-workload-monitoring/hypershift-enable-user-workload-monitoring-ref.yaml", + "owners": { + "approvers": [ + "bryan-cox", + "cblecker", + "csrwng", + "enxebre", + "jparrill", + "mgencur" + ], + "reviewers": [ + "bryan-cox", + "cblecker", + "csrwng", + "enxebre", + "jparrill", + "mgencur" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/hypershift-enable-user-workload-monitoring-ref.yaml b/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/hypershift-enable-user-workload-monitoring-ref.yaml new file mode 100644 index 000000000000..1e3b85a1e44d --- /dev/null +++ b/ci-operator/step-registry/hypershift/enable-user-workload-monitoring/hypershift-enable-user-workload-monitoring-ref.yaml @@ -0,0 +1,37 @@ +ref: + as: hypershift-enable-user-workload-monitoring + from: cli + env: + - name: MONITORING_KUBECONFIG + default: "" + documentation: |- + Kubeconfig of the cluster to enable user workload monitoring on. Defaults + to ${SHARED_DIR}/management_cluster_kubeconfig, which is where the + HyperShift workflows put the management cluster. Point it elsewhere to + reuse this step on a standalone cluster. + - name: UWM_WAIT_ATTEMPTS + default: "60" + documentation: |- + How many 10 second polls to wait for the prometheus-user-workload + StatefulSet to appear after the ConfigMap is written. + - name: UWM_ROLLOUT_TIMEOUT + default: "10m" + documentation: "Timeout passed to \"oc rollout status\" for prometheus-user-workload." + commands: hypershift-enable-user-workload-monitoring-commands.sh + resources: + requests: + cpu: 100m + memory: 200Mi + timeout: 20m0s + documentation: |- + Enables user workload monitoring on a cluster and waits for the user + workload Prometheus to roll out. + + Anything that runs in a namespace the cluster monitoring stack does not own + -- a hosted control plane in "clusters-", an operator shipping its own + ServiceMonitor -- is only scraped once user workload monitoring is on. Any + step that later queries those series needs this to have happened first, so + it is a step of its own rather than a preamble inside one workload. + + The ConfigMap is merged rather than overwritten, and the step is a no-op when + user workload monitoring is already enabled. diff --git a/ci-operator/step-registry/hypershift/etcd-perf/OWNERS b/ci-operator/step-registry/hypershift/etcd-perf/OWNERS deleted file mode 100644 index 588d9076e4d8..000000000000 --- a/ci-operator/step-registry/hypershift/etcd-perf/OWNERS +++ /dev/null @@ -1,15 +0,0 @@ -approvers: -- bryan-cox -- cblecker -- csrwng -- enxebre -- jparrill -- mgencur -options: {} -reviewers: -- bryan-cox -- cblecker -- csrwng -- enxebre -- jparrill -- mgencur diff --git a/ci-operator/step-registry/hypershift/etcd-perf/OWNERS b/ci-operator/step-registry/hypershift/etcd-perf/OWNERS new file mode 120000 index 000000000000..ec405d65a79d --- /dev/null +++ b/ci-operator/step-registry/hypershift/etcd-perf/OWNERS @@ -0,0 +1 @@ +../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/hypershift/etcd-perf/README.md b/ci-operator/step-registry/hypershift/etcd-perf/README.md index 66da920c89c2..46db3ff5b6a0 100644 --- a/ci-operator/step-registry/hypershift/etcd-perf/README.md +++ b/ci-operator/step-registry/hypershift/etcd-perf/README.md @@ -10,69 +10,112 @@ that keys land in the shard they are routed to, and that the shards survive a restart. Those jobs deliberately measure nothing. This one produces numbers and nothing else. -## Why a hosted cluster needs its own step +## Shape + +The chain is deliberately three small steps rather than one script, so that the +only HyperShift-specific part is the plumbing and the workload itself is the +perf and scale team's own step, used unmodified: + +| Step | Owner | What it does | +|---|---|---| +| `hypershift-enable-user-workload-monitoring` | HyperShift | Turns on user workload monitoring on the management cluster | +| `hypershift-hcp-metrics-endpoint` | HyperShift | Resolves the management cluster's Thanos, writes the kube-burner endpoint file | +| `openshift-qe-etcd-density-annotation-churn` | perf and scale | Runs the workload | + +Nothing in this directory contains a metrics query. The queries come from +`metrics-profiles/` in `cloud-bulldozer/e2e-benchmarking`, at the same tag the +`openshift-qe` workload steps resolve, so there is one copy of them and it is +owned by the team that owns the measurements. + +## Why a hosted cluster needs the extra plumbing On a standalone cluster, kube-burner points at one cluster and finds etcd in `openshift-etcd` on that same cluster. On a hosted cluster the two halves are on different clusters: the load lands on the guest API server, but the etcd under -test is a set of pods in `clusters-` on the management cluster. +test is a set of pods in `clusters-` on the management cluster. Two things +follow: -So this step: +- Those pods are in a user namespace, so nothing scrapes their ServiceMonitors + until user workload monitoring is enabled. +- kube-burner's own Prometheus discovery finds the cluster in `KUBECONFIG`, + which is the guest. The series it needs are on the other one, so it is handed + an explicit endpoint instead. -- drives the workload with `${SHARED_DIR}/nested_kubeconfig` (the guest), -- scrapes metrics from the management cluster's Thanos, via a metrics profile - re-scoped from `namespace="openshift-etcd"` to `namespace="clusters-"`, -- enables user workload monitoring on the management cluster first, because - without it nothing scrapes the hosted control plane's ServiceMonitors at all, -- verifies that etcd series are actually arriving before starting the workload, - so a job cannot burn hours producing load and no measurement. +`hypershift-hcp-metrics-endpoint` also verifies that etcd series are actually +arriving before the workload starts, so a job cannot burn hours producing load +and no measurement. -The `kube-burner-ocp-wrapper` in `cloud-bulldozer/e2e-benchmarking` does have a +The `kube-burner-ocp-wrapper` in `e2e-benchmarking` does have a hosted-control-plane mode, but its AWS path is ROSA specific: it derives the -control plane namespace from the `api.openshift.com/name` resource tag and it +control plane namespace from the `api.openshift.com/name` resource tag and expects an Observability Operator route on the management cluster. Neither holds -on the HyperShift CI lanes, which is why this step talks to `kube-burner-ocp` -directly. +on the self-managed HyperShift CI lanes. Its *profiles* are reused; its +discovery is not. ## Per-shard breakdown -Every etcd series in the profile keeps its `pod` label. With sharding enabled -the pod names carry the shard name (`etcd-0`, `etcd-events-0`, `etcd-leases-0`, -...), so `by (pod)` is a per-shard breakdown for free. The shard topology in -effect is also dumped to `hostedcluster-etcd-spec.json` and `etcd-pods.txt` in -the artifact directory. +Not there yet. The upstream `rosa/hosted-cp-metrics.yml` aggregates etcd series +across the control plane namespace without a `by (pod)` breakdown, so a sharded +run currently reports the shards summed together. Because the pod names carry +the shard name (`etcd-0`, `etcd-events-0`, `etcd-leases-0`, ...), splitting them +is a `by (pod)` away — but that belongs in `e2e-benchmarking` next to the rest +of the queries, not in a copy of the profile here. Until that lands, the shard +topology in effect is recorded in `hostedcluster-etcd-spec.json` and +`etcd-statefulsets.txt` in the artifact directory. ## Comparing standard against sharded -The step does not compare topologies. It measures whichever hosted cluster it is -pointed at and records `shardingVariant` in the result metadata. A comparison is -two jobs on the same schedule running the same workload with the same flags, -where one creates the hosted cluster with `HYPERSHIFT_ETCD_SHARDS` set and the -other does not. With `INDEX_TO_ES: "true"` both land in the perf and scale -Elasticsearch and Orion can do the diffing; without it, the numbers are in the -artifact directory under `collected-metrics-/`. +The chain does not compare topologies. It measures whichever hosted cluster it +is pointed at. A comparison is two jobs on the same schedule running the same +workload with the same flags, where one creates the hosted cluster with +`HYPERSHIFT_ETCD_SHARDS` set and the other does not. + +The shard topology the jobs use is the one agreed with the perf and scale team: +three PersistentVolume-backed main etcd members, a three-member +PersistentVolume-backed `leases` shard so that cluster heartbeat signals get +their own highly available shard, and a single EmptyDir-backed `events` shard. + +```yaml +HYPERSHIFT_CP_AVAILABILITY_POLICY: HighlyAvailable # the three main members +HYPERSHIFT_ETCD_SHARDS: |- + name=leases,resources=coordination.k8s.io/leases,replicas=3,storage=PersistentVolume + name=events,resources=/events;events.k8s.io/events,replicas=1,storage=EmptyDir +``` + +`EtcdSharding` is a `TechPreviewNoUpgrade` management cluster feature gate, so +the sharded job also needs `TECH_PREVIEW_NO_UPGRADE: "true"`. ## Workloads The four `etcd-density` sub-workloads are the ones the OpenShift Performance and Scale team used to characterise standard against sharded on HCP: -| Workload | What it stresses | -|---|---| -| `event-storm` | Bulk event creation against a concurrent victim workload | -| `crashloop-flood` | Continuous event pressure through the PATCH/count-increment path, which is the shape most real event storms take | -| `db-quota-pressure` | Large object fill driving the backend toward its storage quota | -| `annotation-churn` | Revision bloat from repeated annotation patching | +| Workload | What it stresses | `openshift-qe` step | +|---|---|---| +| `annotation-churn` | Revision bloat from repeated annotation patching | yes | +| `db-quota-pressure` | Large object fill driving the backend toward its storage quota | yes | +| `crashloop-flood` | Continuous event pressure through the PATCH/count-increment path, which is the shape most real event storms take | not yet | +| `event-storm` | Bulk event creation against a concurrent victim workload | not yet | + +They are added one per pull request, starting with `annotation-churn`. Adding +one is swapping the last ref of the chain, plus writing the `openshift-qe` step +for it if there is not one yet. + +`db-quota-pressure` needs `GC: "false"`, otherwise kube-burner removes the +objects that are supposed to be holding the backend under pressure. Its +`openshift-qe` step already defaults it that way. -Their flag sets differ; pass them through `ETCD_PERF_FLAGS` and check -`kube-burner-ocp etcd-density --help` for what each accepts. +## Results -Note that `db-quota-pressure` needs `GC: "false"`, otherwise kube-burner removes -the objects that are supposed to be holding the backend under pressure. +`collected-metrics-/` in the artifact directory. -## Gating +Not Elasticsearch, yet, and so not Orion. `run.sh` in `e2e-benchmarking` passes +either `--metrics-endpoint` or `--es-server` to kube-burner, never both, and the +hosted control plane series only exist on the endpoint. Its existing +hosted-control-plane mode would resolve that, but it only knows ROSA and ARO; +giving it a self-managed HyperShift branch is what unblocks indexing, and it +belongs in that repo. Until then these jobs are read from their artifacts. -`ETCD_MAX_P99_COMMIT_LATENCY_MS` and `ETCD_MAX_LEADER_CHANGES` are both off by -default. When set they should be set generously: their job is to fail a run that -has gone obviously wrong, not to detect regressions. Regression detection is -Orion's, over many runs. +No credential handling lives in this directory. The one Elasticsearch password +in the chain is read by the `openshift-qe` workload step, which already owned +that code before this chain existed. diff --git a/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-ref.metadata.json b/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-chain.metadata.json similarity index 78% rename from ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-ref.metadata.json rename to ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-chain.metadata.json index f538ae53ed47..7a1a76aac3bb 100644 --- a/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-ref.metadata.json +++ b/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-chain.metadata.json @@ -1,5 +1,5 @@ { - "path": "hypershift/etcd-perf/hypershift-etcd-perf-ref.yaml", + "path": "hypershift/etcd-perf/hypershift-etcd-perf-chain.yaml", "owners": { "approvers": [ "bryan-cox", diff --git a/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-chain.yaml b/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-chain.yaml new file mode 100644 index 000000000000..cc2732c016f5 --- /dev/null +++ b/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-chain.yaml @@ -0,0 +1,34 @@ +chain: + as: hypershift-etcd-perf + steps: + - ref: hypershift-enable-user-workload-monitoring + - ref: hypershift-hcp-metrics-endpoint + - ref: openshift-qe-etcd-density-annotation-churn + documentation: |- + Runs a kube-burner-ocp "etcd-density" workload against a HyperShift hosted + cluster and collects the hosted control plane's etcd and kube-apiserver + metrics from the management cluster. + + The chain is three steps because each of them is useful on its own: + + - hypershift-enable-user-workload-monitoring turns on the monitoring the + hosted control plane namespace needs to be scraped at all. + - hypershift-hcp-metrics-endpoint points kube-burner at the management + cluster's Thanos, with the query set taken from e2e-benchmarking. + - openshift-qe-etcd-density-annotation-churn is the perf and scale team's + own workload step, used unmodified, so the HCP numbers and the standalone + OpenShift numbers come from the same workload definition. + + Shard topologies are not compared by the chain. Comparison is done by + running it twice, once against a hosted cluster created with + HYPERSHIFT_ETCD_SHARDS and once without, and diffing the two result sets. + + Other etcd-density variants (crashloop-flood, db-quota-pressure, + event-storm) are added by swapping the last ref, one variant per pull + request. db-quota-pressure already has an openshift-qe step; the other two + do not yet. + + Requires a hosted cluster to already exist, i.e. the hypershift-aws-create + chain or an equivalent must have run and written + ${SHARED_DIR}/cluster-name, ${SHARED_DIR}/nested_kubeconfig and + ${SHARED_DIR}/management_cluster_kubeconfig. diff --git a/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-commands.sh b/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-commands.sh deleted file mode 100644 index bcd189a50380..000000000000 --- a/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-commands.sh +++ /dev/null @@ -1,450 +0,0 @@ -#!/bin/bash - -set -euo pipefail -# Do not enable `set -x`. This script mints a Prometheus bearer token and reads -# Elasticsearch credentials; tracing would put both in the CI logs. - -MC_KUBECONFIG="${SHARED_DIR}/management_cluster_kubeconfig" -GUEST_KUBECONFIG="${SHARED_DIR}/nested_kubeconfig" - -for f in "${MC_KUBECONFIG}" "${GUEST_KUBECONFIG}" "${SHARED_DIR}/cluster-name"; do - if [[ ! -s "${f}" ]]; then - echo "FATAL: ${f} is missing or empty; this step needs a hosted cluster to already exist." - exit 1 - fi -done - -CLUSTER_NAME="$(cat "${SHARED_DIR}/cluster-name")" -# HyperShift puts the hosted control plane in "-", -# and every CI workflow that feeds this step creates the HostedCluster in "clusters". -HCP_NAMESPACE="clusters-${CLUSTER_NAME}" - -mc() { oc --kubeconfig="${MC_KUBECONFIG}" "$@"; } - -echo "Hosted cluster: ${CLUSTER_NAME}" -echo "Hosted control plane ns: ${HCP_NAMESPACE}" -echo "Workload: etcd-density ${ETCD_PERF_WORKLOAD}" -echo "Shard topology variant: ${ETCD_PERF_VARIANT}" - -if ! mc get namespace "${HCP_NAMESPACE}" >/dev/null 2>&1; then - echo "FATAL: namespace ${HCP_NAMESPACE} does not exist on the management cluster." - exit 1 -fi - -echo -echo "=== etcd shards in ${HCP_NAMESPACE} ===" -# Recorded so the artifacts say which topology produced them. With sharding the -# pod names carry the shard name (etcd-0, etcd-events-0, ...), which is what -# makes the "by (pod)" breakdown in the metrics profile a per-shard breakdown. -mc get statefulset -n "${HCP_NAMESPACE}" -l app=etcd -o wide 2>/dev/null || true -mc get pods -n "${HCP_NAMESPACE}" -l app=etcd -o name 2>/dev/null | tee "${ARTIFACT_DIR}/etcd-pods.txt" || true -mc get hostedcluster -n clusters "${CLUSTER_NAME}" -o jsonpath='{.spec.etcd}' \ - > "${ARTIFACT_DIR}/hostedcluster-etcd-spec.json" 2>/dev/null || true -echo - -# --------------------------------------------------------------------------- -# Management cluster monitoring -# -# The hosted control plane's etcd and kube-apiserver run as ordinary workloads -# in ${HCP_NAMESPACE}, so their ServiceMonitors are only scraped if user -# workload monitoring is on. The management cluster here is ephemeral and -# per-job, so turning it on is safe, but still only do it if it is off. -# --------------------------------------------------------------------------- -enable_user_workload_monitoring() { - local existing - if ! existing="$(mc -n openshift-monitoring get configmap cluster-monitoring-config \ - -o jsonpath='{.data.config\.yaml}' 2>/dev/null)"; then - echo "Creating cluster-monitoring-config with user workload monitoring enabled" - mc apply -f - <<'EOF' -apiVersion: v1 -kind: ConfigMap -metadata: - name: cluster-monitoring-config - namespace: openshift-monitoring -data: - config.yaml: | - enableUserWorkload: true -EOF - return - fi - - if grep -Eq '^[[:space:]]*enableUserWorkload:[[:space:]]*true' <<< "${existing}"; then - echo "User workload monitoring is already enabled" - return - fi - - echo "Appending enableUserWorkload to the existing cluster-monitoring-config" - printf '%s\nenableUserWorkload: true\n' "${existing}" > /tmp/cluster-monitoring-config.yaml - mc -n openshift-monitoring create configmap cluster-monitoring-config \ - --from-file=config.yaml=/tmp/cluster-monitoring-config.yaml \ - --dry-run=client -o yaml | mc replace -f - -} - -enable_user_workload_monitoring - -echo "Waiting for the user workload Prometheus to come up" -for _ in $(seq 1 60); do - if mc -n openshift-user-workload-monitoring get statefulset prometheus-user-workload >/dev/null 2>&1; then - break - fi - sleep 10 -done -mc -n openshift-user-workload-monitoring rollout status statefulset/prometheus-user-workload --timeout=10m - -# --------------------------------------------------------------------------- -# Thanos endpoint and token -# --------------------------------------------------------------------------- -PROM_SA="hypershift-etcd-perf" -mc -n openshift-monitoring create serviceaccount "${PROM_SA}" --dry-run=client -o yaml | mc apply -f - -mc adm policy add-cluster-role-to-user cluster-monitoring-view \ - -z "${PROM_SA}" -n openshift-monitoring >/dev/null - -THANOS_HOST="$(mc -n openshift-monitoring get route thanos-querier -o jsonpath='{.spec.host}')" -if [[ -z "${THANOS_HOST}" ]]; then - echo "FATAL: could not resolve the thanos-querier route on the management cluster." - exit 1 -fi -THANOS_URL="https://${THANOS_HOST}" -PROM_TOKEN="$(mc -n openshift-monitoring create token "${PROM_SA}" --duration=8h)" -echo "Thanos endpoint: ${THANOS_URL}" - -# The management cluster's ingress certificate is not one this container trusts, -# hence -k here and skipTLSVerify in the metrics endpoint below. -promq() { - curl -sSk --get --data-urlencode "query=$1" \ - -H "Authorization: Bearer ${PROM_TOKEN}" \ - "${THANOS_URL}/api/v1/query" -} - -echo "Waiting ${SCRAPE_SETTLE_SECONDS}s for the hosted control plane targets to be scraped" -sleep "${SCRAPE_SETTLE_SECONDS}" - -series_count="$(promq "count(etcd_server_has_leader{namespace=\"${HCP_NAMESPACE}\"})" \ - | jq -r '.data.result[0].value[1] // "0"')" -if [[ "${series_count}" == "0" ]]; then - echo "FATAL: Thanos returns no etcd metrics for namespace ${HCP_NAMESPACE}." - echo "Without them this run would produce a workload but no measurement, so fail now" - echo "rather than burn the cluster time. Check that the hosted control plane's etcd" - echo "ServiceMonitors exist and that user workload monitoring came up." - mc get servicemonitor -n "${HCP_NAMESPACE}" 2>&1 | head -20 || true - exit 1 -fi -echo "Scraping ${series_count} etcd member(s) in ${HCP_NAMESPACE}" - -# --------------------------------------------------------------------------- -# Metrics profile -# -# This is the upstream kube-burner-ocp etcd-density profile re-scoped for a -# hosted control plane: every series is confined to ${HCP_NAMESPACE} on the -# management cluster instead of openshift-etcd, and the etcd series keep their -# pod label so each shard is reported on its own. The metric set is the one the -# perf and scale team characterised standard vs sharded with. -# --------------------------------------------------------------------------- -PROFILE=/tmp/hcp-etcd-density-metrics.yml -cat > "${PROFILE}" < 0 - metricName: etcdCompaction -- query: etcd_debugging_mvcc_db_compaction_total_duration_milliseconds_sum{namespace="${HCP_NAMESPACE}"} - metricName: etcdCompactionDurationTotal -- query: delta(etcd_disk_backend_defrag_duration_seconds_sum{namespace="${HCP_NAMESPACE}"}[1m:30s])/2 > 0 - metricName: etcdDefrag -- query: sum by (pod) (rate(etcd_debugging_snap_save_total_duration_seconds_sum{namespace="${HCP_NAMESPACE}"}[2m])) - metricName: etcdSnapshotDuration - -# Raft -- query: sum by (pod) (rate(etcd_server_proposals_committed_total{namespace="${HCP_NAMESPACE}"}[2m])) - metricName: etcdProposalsCommittedRate -- query: sum by (pod) (rate(etcd_server_proposals_applied_total{namespace="${HCP_NAMESPACE}"}[2m])) - metricName: etcdProposalsAppliedRate -- query: sum by (pod) (rate(etcd_server_proposals_failed_total{namespace="${HCP_NAMESPACE}"}[2m])) - metricName: etcdProposalsFailedRate -- query: sum by (pod) (etcd_server_proposals_pending{namespace="${HCP_NAMESPACE}"}) - metricName: etcdProposalsPending -- query: sum by (pod) (rate(etcd_server_leader_changes_seen_total{namespace="${HCP_NAMESPACE}"}[2m])) - metricName: etcdLeaderChangesRate -- query: max by (pod) (etcd_server_has_leader{namespace="${HCP_NAMESPACE}"}) - metricName: etcdHasLeader - -# Slow operations -- query: delta(etcd_server_slow_apply_total{namespace="${HCP_NAMESPACE}"}[2m]) - metricName: etcdSlowApply -- query: delta(etcd_server_slow_read_indexes_total{namespace="${HCP_NAMESPACE}"}[2m]) - metricName: etcdSlowReadIndexes - -# Key space. With sharding this splits across pods, which is the whole point. -- query: etcd_debugging_mvcc_keys_total{namespace="${HCP_NAMESPACE}"} - metricName: etcdKeys -- query: etcd_debugging_mvcc_db_compaction_keys_total{namespace="${HCP_NAMESPACE}"} - metricName: etcdCompactedKeys -- query: rate(etcd_mvcc_put_total{namespace="${HCP_NAMESPACE}"}[2m]) - metricName: etcdPutRate -- query: rate(etcd_mvcc_delete_total{namespace="${HCP_NAMESPACE}"}[2m]) - metricName: etcdDeleteRate - -# Container resources. Sharding buys latency at the cost of running more etcd -# members, and the measured memory overhead was large, so track it per pod. -- query: (sum(irate(container_cpu_usage_seconds_total{namespace="${HCP_NAMESPACE}", container="etcd"}[2m]) * 100) by (pod)) > 0 - metricName: containerCPU -- query: sum(container_memory_working_set_bytes{namespace="${HCP_NAMESPACE}", container="etcd"}) by (pod) - metricName: containerMemory -- query: sum(container_memory_working_set_bytes{namespace="${HCP_NAMESPACE}"}) by (pod, container) - metricName: hcpContainerMemory - -# Management cluster node pressure. Not filtered to the nodes hosting this -# control plane: the node list is written to the artifact directory separately -# so the series can be narrowed during analysis. -- query: rate(node_vmstat_pgmajfault[2m]) - metricName: nodeMajorPageFaults - -# Hosted kube-apiserver. This is what the guest cluster actually feels when -# etcd slows down, and it is the number a shard routing change should move. -- query: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{namespace="${HCP_NAMESPACE}", subresource!="log", verb!~"WATCH|WATCHLIST|PROXY"}[2m])) by (verb, le)) - metricName: requestDuration99thByVerb -- query: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{namespace="${HCP_NAMESPACE}", subresource!="log", verb!~"WATCH|WATCHLIST|PROXY"}[2m])) by (resource, le)) - metricName: requestDuration99thByResource -- query: sum(rate(apiserver_request_total{namespace="${HCP_NAMESPACE}", verb!="WATCH"}[2m])) by (verb, resource, code) > 0 - metricName: APIRequestRate -- query: sum(apiserver_current_inflight_requests{namespace="${HCP_NAMESPACE}"}) by (request_kind) - metricName: requestInFlight -- query: sum(rate(apiserver_request_terminations_total{namespace="${HCP_NAMESPACE}"}[2m])) by (verb) - metricName: requestRateDropped -- query: topk(20, max by (resource) (apiserver_storage_objects{namespace="${HCP_NAMESPACE}"})) - metricName: etcdStorageObjects - -# Start and end snapshots -- query: etcd_mvcc_db_total_size_in_bytes{namespace="${HCP_NAMESPACE}"} - metricName: etcdDBTotalSizeSnapshot - instant: true - captureStart: true -- query: etcd_debugging_mvcc_keys_total{namespace="${HCP_NAMESPACE}"} - metricName: etcdKeysSnapshot - instant: true - captureStart: true -- query: etcd_server_proposals_committed_total{namespace="${HCP_NAMESPACE}"} - metricName: etcdProposalsCommitted - instant: true - captureStart: true -- query: etcd_server_slow_apply_total{namespace="${HCP_NAMESPACE}"} - metricName: etcdSlowApplyTotal - instant: true - captureStart: true -- query: etcd_server_leader_changes_seen_total{namespace="${HCP_NAMESPACE}"} - metricName: etcdLeaderChanges - instant: true - captureStart: true -- query: etcd_debugging_mvcc_db_compaction_total_duration_milliseconds_sum{namespace="${HCP_NAMESPACE}"} - metricName: etcdCompaction-raw - instant: true - captureStart: true -- query: sum(container_memory_working_set_bytes{namespace="${HCP_NAMESPACE}", container="etcd"}) by (pod) - metricName: containerMemorySnapshot - instant: true - captureStart: true -EOF -cp "${PROFILE}" "${ARTIFACT_DIR}/hcp-etcd-density-metrics.yml" - -# Nodes of the management cluster that are actually hosting this control plane, -# for narrowing the unfiltered node level series during analysis. -mc get pods -n "${HCP_NAMESPACE}" -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' \ - | sort -u | grep -v '^$' > "${ARTIFACT_DIR}/hcp-management-nodes.txt" || true - -# --------------------------------------------------------------------------- -# Metrics endpoints -# -# An endpoint with no indexer collects nothing that survives the run, so there -# is always a "local" one writing files that end up in the artifact directory. -# Indexing to Elasticsearch, when asked for, is a second endpoint over the same -# Thanos rather than a replacement, so a run is always inspectable by hand even -# when Orion is also getting the data. -# -# This file lives in /tmp and is never copied to ARTIFACT_DIR: it embeds the -# bearer token and, when indexing is on, the Elasticsearch credentials. -# --------------------------------------------------------------------------- -UUID="$(uuidgen)" -METRICS_DIR="/tmp/collected-metrics-${UUID}" -ENDPOINT_FILE=/tmp/metrics-endpoint.yml -{ - echo "- endpoint: ${THANOS_URL}" - echo " token: ${PROM_TOKEN}" - echo " skipTLSVerify: true" - echo " step: ${METRICS_STEP}" - echo " metrics:" - echo " - ${PROFILE}" - echo " indexer:" - echo " type: local" - echo " metricsDirectory: ${METRICS_DIR}" -} > "${ENDPOINT_FILE}" - -if [[ "${INDEX_TO_ES}" == "true" ]]; then - ES_HOST="search-ocp-qe-perf-scale-test-elk-hcm7wtsqpxy7xogbu72bor4uve.us-east-1.es.amazonaws.com" - if [[ -e "${ES_SECRETS_PATH}/host" ]]; then - ES_HOST="$(cat "${ES_SECRETS_PATH}/host")" - fi - ES_USERNAME="$(cat "${ES_SECRETS_PATH}/username")" - ES_PASSWORD="$(cat "${ES_SECRETS_PATH}/password")" - { - echo "- endpoint: ${THANOS_URL}" - echo " token: ${PROM_TOKEN}" - echo " skipTLSVerify: true" - echo " step: ${METRICS_STEP}" - echo " metrics:" - echo " - ${PROFILE}" - echo " indexer:" - echo " type: opensearch" - echo " esServers: [\"https://${ES_USERNAME}:${ES_PASSWORD}@${ES_HOST}\"]" - echo " defaultIndex: ${ES_INDEX}" - echo " insecureSkipVerify: true" - } >> "${ENDPOINT_FILE}" - echo "Results will also be indexed to Elasticsearch under uuid ${UUID}" -else - echo "INDEX_TO_ES is false; results stay in the artifact directory only" -fi - -# Metadata that makes a result set identifiable after the fact. Consumed by -# kube-burner as extra document fields, and kept as an artifact either way. -cat > /tmp/user-metadata.yaml < b)}'; then - FAILURES+=("peak p99 etcd backend commit latency was ${peak_ms}ms, above the ${ETCD_MAX_P99_COMMIT_LATENCY_MS}ms limit") - fi -fi - -if [[ -n "${ETCD_MAX_LEADER_CHANGES}" ]]; then - changes="$(promq "max(increase(etcd_server_leader_changes_seen_total{namespace=\"${HCP_NAMESPACE}\"}[2h]))" \ - | jq -r '.data.result[0].value[1] // "0"')" - printf 'etcd leader changes during the run: %.0f (limit %s)\n' "${changes}" "${ETCD_MAX_LEADER_CHANGES}" - if awk -v a="${changes}" -v b="${ETCD_MAX_LEADER_CHANGES}" 'BEGIN{exit !(a > b)}'; then - FAILURES+=("etcd saw ${changes} leader changes, above the ${ETCD_MAX_LEADER_CHANGES} limit") - fi -fi - -TESTCASE_NAME="etcd-density ${ETCD_PERF_WORKLOAD} (${ETCD_PERF_VARIANT})" -JUNIT="${ARTIFACT_DIR}/junit_hypershift_etcd_perf.xml" -if [[ ${RUN_EXIT_CODE} -ne 0 ]]; then - FAILURES+=("kube-burner-ocp exited with ${RUN_EXIT_CODE}") -fi - -if [[ ${#FAILURES[@]} -eq 0 ]]; then - cat > "${JUNIT}" < - - - -EOF - exit 0 -fi - -{ - echo '' - echo '' - echo " " - echo ' ' - echo ' ' - echo '' -} > "${JUNIT}" - -printf 'FAILED: %s\n' "${FAILURES[@]}" -exit 1 diff --git a/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-ref.yaml b/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-ref.yaml deleted file mode 100644 index 61e087d7d8e7..000000000000 --- a/ci-operator/step-registry/hypershift/etcd-perf/hypershift-etcd-perf-ref.yaml +++ /dev/null @@ -1,112 +0,0 @@ -ref: - as: hypershift-etcd-perf - from_image: - namespace: ci - name: ocp-qe-perfscale-ci - tag: latest - cli: latest - env: - - name: ETCD_PERF_WORKLOAD - default: "event-storm" - documentation: |- - Which "kube-burner-ocp etcd-density" sub-workload to run. One of - event-storm, crashloop-flood, db-quota-pressure, annotation-churn. - These are the four workloads the OpenShift Performance and Scale team - used for the standard-vs-sharded etcd characterisation on HCP. - - name: ETCD_PERF_FLAGS - default: "" - documentation: |- - Workload specific flags appended verbatim to the kube-burner-ocp - invocation, e.g. "--iterations=900 --event-iterations=2000". - See "kube-burner-ocp etcd-density --help" for the flag set; - it differs per workload. - - name: ETCD_PERF_VARIANT - default: "standard" - documentation: |- - Free form label recorded alongside the results so the sharded run can be - told apart from the standard baseline run when the two are compared. - Conventionally "standard" or "sharded"; it has no effect on what runs. - - name: QPS - default: "20" - documentation: "Client side QPS for kube-burner. 20 matches the perf and scale baseline." - - name: BURST - default: "40" - documentation: "Client side burst for kube-burner. 40 matches the perf and scale baseline." - - name: GC - default: "true" - documentation: |- - Whether kube-burner deletes the objects it created. Set to false for - db-quota-pressure, which has to retain its objects to keep the backend - under storage pressure for the duration of the measurement. - - name: KUBE_BURNER_VERSION - default: "1.12.6" - documentation: "kube-burner-ocp release to download." - - name: KUBE_BURNER_URL - default: "" - documentation: "Full URL of a kube-burner-ocp tarball, overriding KUBE_BURNER_VERSION." - - name: METRICS_STEP - default: "30s" - documentation: "Prometheus range query resolution used when collecting the metrics profile." - - name: SCRAPE_SETTLE_SECONDS - default: "180" - documentation: |- - How long to wait after enabling user workload monitoring before checking - that the hosted control plane etcd targets are actually being scraped. - - name: INDEX_TO_ES - default: "false" - documentation: |- - When true, index the collected metrics into the perf and scale - Elasticsearch so Orion can do run-over-run regression detection. - When false the metrics are only written to the artifact directory. - - name: ES_SECRETS_PATH - default: "/secret" - documentation: "Directory holding the Elasticsearch username, password and optional host files." - - name: ES_INDEX - default: "ripsaw-kube-burner" - documentation: "Elasticsearch index to write the collected metrics to." - - name: ETCD_MAX_P99_COMMIT_LATENCY_MS - default: "" - documentation: |- - Optional hard gate. If set, the step fails when the peak 99th percentile - etcd backend commit latency observed during the run exceeds this many - milliseconds. Leave empty to collect data without gating; the intended - use is a deliberately generous bound that only catches gross regressions, - with trend detection left to Orion over the indexed results. - - name: ETCD_MAX_LEADER_CHANGES - default: "" - documentation: |- - Optional hard gate. If set, the step fails when any etcd shard saw more - than this many leader changes during the run. Leave empty to disable. - commands: hypershift-etcd-perf-commands.sh - credentials: - - collection: ocp-perfscale-cpt - group: ocp-qe-perfscale-es-creds - mount_path: /secret - resources: - requests: - cpu: 100m - memory: 200Mi - timeout: 4h0m0s - grace_period: 10m0s - documentation: |- - Runs one of the kube-burner-ocp "etcd-density" workloads against a - HyperShift hosted cluster and collects the etcd and kube-apiserver metrics - of that cluster's hosted control plane. - - The interesting property of a hosted cluster is that the load is applied to - the guest API server while the etcd being measured runs as pods in the - "clusters-" namespace of the management cluster. So this step drives - the workload with the guest kubeconfig and scrapes metrics from the - management cluster's Thanos, using a metrics profile scoped to the hosted - control plane namespace and broken down "by (pod)" so that each etcd shard - is reported separately. - - It does not compare shard topologies by itself. Comparison is done by - running the same workload twice, once against a hosted cluster created with - "--etcd-shard" flags and once without, and diffing the two result sets -- - either by hand from the artifact directory or, when INDEX_TO_ES is true, - with Orion over the indexed metrics. - - Requires a hosted cluster to already exist, i.e. the hypershift-aws-create - chain (or an equivalent) must have run and written ${SHARED_DIR}/cluster-name - and ${SHARED_DIR}/nested_kubeconfig. diff --git a/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/OWNERS b/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/OWNERS new file mode 120000 index 000000000000..ec405d65a79d --- /dev/null +++ b/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/OWNERS @@ -0,0 +1 @@ +../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/hypershift-hcp-metrics-endpoint-commands.sh b/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/hypershift-hcp-metrics-endpoint-commands.sh new file mode 100755 index 000000000000..12319dc8d26f --- /dev/null +++ b/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/hypershift-hcp-metrics-endpoint-commands.sh @@ -0,0 +1,155 @@ +#!/bin/bash + +set -euo pipefail +# Do not enable `set -x`. This step mints a Prometheus bearer token; tracing +# would put it in the CI logs. + +MC_KUBECONFIG="${SHARED_DIR}/management_cluster_kubeconfig" +for f in "${MC_KUBECONFIG}" "${SHARED_DIR}/cluster-name"; do + if [[ ! -s "${f}" ]]; then + echo "FATAL: ${f} is missing or empty; this step needs a hosted cluster to already exist." + exit 1 + fi +done + +CLUSTER_NAME="$(cat "${SHARED_DIR}/cluster-name")" +# HyperShift puts the hosted control plane in "-", +# and every CI workflow that feeds this step creates the HostedCluster in "clusters". +HCP_NAMESPACE="clusters-${CLUSTER_NAME}" + +mc() { oc --kubeconfig="${MC_KUBECONFIG}" "$@"; } + +echo "Hosted cluster: ${CLUSTER_NAME}" +echo "Hosted control plane ns: ${HCP_NAMESPACE}" + +if ! mc get namespace "${HCP_NAMESPACE}" >/dev/null 2>&1; then + echo "FATAL: namespace ${HCP_NAMESPACE} does not exist on the management cluster." + exit 1 +fi + +# Record which etcd topology produced the results. With sharding the pod names +# carry the shard name (etcd-0, etcd-events-0, ...), so this is also the record +# of which shards the metrics below are coming from. +mc get statefulset -n "${HCP_NAMESPACE}" -l app=etcd -o wide 2>&1 \ + | tee "${ARTIFACT_DIR}/etcd-statefulsets.txt" || true +mc get hostedcluster -n clusters "${CLUSTER_NAME}" -o jsonpath='{.spec.etcd}' \ + > "${ARTIFACT_DIR}/hostedcluster-etcd-spec.json" 2>/dev/null || true + +# --------------------------------------------------------------------------- +# Metrics profiles +# +# Taken from the kube-burner-ocp wrapper in e2e-benchmarking rather than +# written here, so the queries stay owned by the perf and scale team and this +# repo does not carry a second copy that drifts. The profiles are copied into +# SHARED_DIR because /tmp does not survive into the step that runs the +# workload. +# --------------------------------------------------------------------------- +REPO_URL="https://github.com/cloud-bulldozer/e2e-benchmarking" +if [[ "${E2E_VERSION}" == "default" ]]; then + E2E_REF="$(git ls-remote --tags "${REPO_URL}.git" \ + | awk -F'refs/tags/' '{print $2}' | grep -v '\^{}' | sort -V | tail -n1)" +else + E2E_REF="${E2E_VERSION}" +fi +echo "Using e2e-benchmarking ${E2E_REF} for the metrics profiles" +git clone "${REPO_URL}" --branch "${E2E_REF}" --depth 1 /tmp/e2e-benchmarking + +WRAPPER=/tmp/e2e-benchmarking/workloads/kube-burner-ocp-wrapper +PROFILE_DIR="${SHARED_DIR}/hcp-metrics-profiles" +mkdir -p "${PROFILE_DIR}" + +for profile in ${HCP_METRICS_PROFILES}; do + src="${WRAPPER}/metrics-profiles/${profile}" + if [[ ! -s "${src}" ]]; then + echo "FATAL: ${profile} does not exist in e2e-benchmarking ${E2E_REF}." + exit 1 + fi + # The upstream profiles are written for ROSA, where the hosted control plane + # namespace is "-" and the queries match it with + # namespace=~".+{{.HCP_NAMESPACE}}". Self-managed HyperShift uses + # "clusters-", so binding the placeholder to the cluster name keeps the + # upstream regex correct without editing the queries. + sed "s|{{\.HCP_NAMESPACE}}|${CLUSTER_NAME}|g" "${src}" > "${PROFILE_DIR}/$(basename "${profile}")" +done +cp -r "${PROFILE_DIR}" "${ARTIFACT_DIR}/hcp-metrics-profiles" + +# --------------------------------------------------------------------------- +# Thanos endpoint and token +# +# thanos-querier rather than prometheus-k8s: the hosted control plane series are +# collected by the user workload Prometheus, and thanos-querier is the endpoint +# that serves both stacks. +# --------------------------------------------------------------------------- +PROM_SA="hypershift-hcp-metrics" +mc -n openshift-monitoring create serviceaccount "${PROM_SA}" --dry-run=client -o yaml | mc apply -f - +mc adm policy add-cluster-role-to-user cluster-monitoring-view \ + -z "${PROM_SA}" -n openshift-monitoring >/dev/null + +THANOS_HOST="$(mc -n openshift-monitoring get route thanos-querier -o jsonpath='{.spec.host}')" +if [[ -z "${THANOS_HOST}" ]]; then + echo "FATAL: could not resolve the thanos-querier route on the management cluster." + exit 1 +fi +THANOS_URL="https://${THANOS_HOST}" +PROM_TOKEN="$(mc -n openshift-monitoring create token "${PROM_SA}" --duration="${PROM_TOKEN_DURATION}")" +echo "Thanos endpoint: ${THANOS_URL}" + +# The management cluster's ingress certificate is not one this container trusts, +# hence -k here and skipTLSVerify in the endpoint file below. +promq() { + curl --fail --silent --show-error -k --get --data-urlencode "query=$1" \ + -H "Authorization: Bearer ${PROM_TOKEN}" \ + "${THANOS_URL}/api/v1/query" +} + +echo "Waiting ${SCRAPE_SETTLE_SECONDS}s for the hosted control plane targets to be scraped" +sleep "${SCRAPE_SETTLE_SECONDS}" + +# Fail now rather than after a multi-hour workload that produced no measurement. +# A query that errors, returns no result, or returns zero members all mean the +# same thing here: there is nothing to measure. +if ! response="$(promq "count(etcd_server_has_leader{namespace=\"${HCP_NAMESPACE}\"})")"; then + echo "FATAL: the Thanos query failed. The endpoint or the token is wrong, or" + echo "thanos-querier on the management cluster is not serving queries." + exit 1 +fi +series_count="$(jq -r '.data.result[0].value[1] // empty' <<< "${response}")" +if [[ -z "${series_count}" || "${series_count}" == "0" ]]; then + echo "FATAL: Thanos returns no etcd metrics for namespace ${HCP_NAMESPACE}." + echo "Check that the hosted control plane's etcd ServiceMonitors exist and that" + echo "user workload monitoring came up (hypershift-enable-user-workload-monitoring)." + mc get servicemonitor -n "${HCP_NAMESPACE}" 2>&1 | head -20 || true + exit 1 +fi +echo "Scraping ${series_count} etcd member(s) in ${HCP_NAMESPACE}" + +# --------------------------------------------------------------------------- +# Endpoint file +# +# Written to SHARED_DIR, never to ARTIFACT_DIR: it carries the bearer token. +# The indexer is local so the series land in the workload step's working +# directory and from there in the artifacts; indexing to Elasticsearch stays +# with the openshift-qe workload step, which already owns those credentials. +# --------------------------------------------------------------------------- +ENDPOINT_FILE="${SHARED_DIR}/hcp-metrics-endpoint.yml" +{ + echo "- endpoint: ${THANOS_URL}" + echo " token: ${PROM_TOKEN}" + echo " skipTLSVerify: true" + echo " step: ${METRICS_STEP}" + echo " metrics:" + for profile in ${HCP_METRICS_PROFILES}; do + echo " - ${PROFILE_DIR}/$(basename "${profile}")" + done + echo " indexer:" + echo " type: local" + # kube-burner renders the endpoint file as a template over the environment, + # the same way the upstream metrics-endpoint.yml uses {{.ES_SERVER}}. UUID is + # exported by the openshift-qe workload step, and the directory name is the + # one that step copies into ARTIFACT_DIR when the run finishes. + echo " metricsDirectory: collected-metrics-{{.UUID}}" +} > "${ENDPOINT_FILE}" +chmod 600 "${ENDPOINT_FILE}" + +echo "${HCP_NAMESPACE}" > "${SHARED_DIR}/hcp-namespace" +echo "Wrote ${ENDPOINT_FILE}" diff --git a/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/hypershift-hcp-metrics-endpoint-ref.metadata.json b/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/hypershift-hcp-metrics-endpoint-ref.metadata.json new file mode 100644 index 000000000000..2fbf1061e83c --- /dev/null +++ b/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/hypershift-hcp-metrics-endpoint-ref.metadata.json @@ -0,0 +1,21 @@ +{ + "path": "hypershift/hcp-metrics-endpoint/hypershift-hcp-metrics-endpoint-ref.yaml", + "owners": { + "approvers": [ + "bryan-cox", + "cblecker", + "csrwng", + "enxebre", + "jparrill", + "mgencur" + ], + "reviewers": [ + "bryan-cox", + "cblecker", + "csrwng", + "enxebre", + "jparrill", + "mgencur" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/hypershift-hcp-metrics-endpoint-ref.yaml b/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/hypershift-hcp-metrics-endpoint-ref.yaml new file mode 100644 index 000000000000..e3f448ec21e9 --- /dev/null +++ b/ci-operator/step-registry/hypershift/hcp-metrics-endpoint/hypershift-hcp-metrics-endpoint-ref.yaml @@ -0,0 +1,71 @@ +ref: + as: hypershift-hcp-metrics-endpoint + # Same image as the openshift-qe workload steps, so the e2e-benchmarking + # checkout this step does and the one the workload step does are identical. + from_image: + namespace: ci + name: ocp-qe-perfscale-ci + tag: latest + cli: latest + env: + - name: E2E_VERSION + default: "default" + documentation: |- + e2e-benchmarking tag to take the metrics profiles from. "default" resolves + to the latest tag, which is what the openshift-qe workload steps do, so + the profiles and the wrapper that consumes them stay on the same release. + - name: HCP_METRICS_PROFILES + default: "rosa/hosted-cp-metrics.yml" + documentation: |- + Space separated list of metrics profiles to take from the kube-burner-ocp + wrapper's "metrics-profiles" directory in e2e-benchmarking. The default is + the hosted control plane profile: etcd, kube-apiserver and OVN series for + the control plane namespace. The "rosa" directory is the hosted control + plane profile for OpenShift-flavoured management clusters; self-managed + HyperShift reuses it, the difference between the two products is the + namespace layout, which this step binds for you. + - name: METRICS_STEP + default: "30s" + documentation: "Prometheus range query resolution used when collecting the profiles." + - name: SCRAPE_SETTLE_SECONDS + default: "180" + documentation: |- + How long to wait before checking that the hosted control plane etcd + targets are actually being scraped. Enabling user workload monitoring and + the first scrape of a newly created namespace are not instantaneous. + - name: PROM_TOKEN_DURATION + default: "8h" + documentation: |- + Lifetime of the Thanos bearer token. Must outlast the workload step that + consumes the endpoint file. + commands: hypershift-hcp-metrics-endpoint-commands.sh + resources: + requests: + cpu: 100m + memory: 200Mi + timeout: 30m0s + documentation: |- + Prepares ${SHARED_DIR}/hcp-metrics-endpoint.yml so that a kube-burner + workload driven against a hosted cluster collects its control plane metrics + from the management cluster. + + A hosted cluster is split: the workload is applied to the guest API server, + but the etcd and kube-apiserver being measured run as pods in the + "clusters-" namespace of the management cluster. kube-burner's own + Prometheus discovery finds the cluster in KUBECONFIG, which is the wrong one + of the two, so this step resolves the management cluster's thanos-querier + route, mints a cluster-monitoring-view token for it, and writes an endpoint + file the workload step picks up. + + The queries are not written here. They are copied from the kube-burner-ocp + wrapper in cloud-bulldozer/e2e-benchmarking, the same source and the same + tag the openshift-qe workload steps use, with the hosted control plane + namespace bound for self-managed HyperShift. Changes to what is measured + belong in that repo. + + Requires hypershift-enable-user-workload-monitoring to have run, and a + hosted cluster to exist: ${SHARED_DIR}/cluster-name and + ${SHARED_DIR}/management_cluster_kubeconfig must be present. + + The endpoint file carries a bearer token. It is written to SHARED_DIR with + mode 600 and is never copied to ARTIFACT_DIR. diff --git a/ci-operator/step-registry/openshift-qe/etcd-density-annotation-churn/openshift-qe-etcd-density-annotation-churn-commands.sh b/ci-operator/step-registry/openshift-qe/etcd-density-annotation-churn/openshift-qe-etcd-density-annotation-churn-commands.sh index d83a6bf37dc9..a354a4518058 100755 --- a/ci-operator/step-registry/openshift-qe/etcd-density-annotation-churn/openshift-qe-etcd-density-annotation-churn-commands.sh +++ b/ci-operator/step-registry/openshift-qe/etcd-density-annotation-churn/openshift-qe-etcd-density-annotation-churn-commands.sh @@ -5,6 +5,22 @@ set -o pipefail # Do not enable `set -x`: these scripts read and export Elasticsearch # credentials (ES_USERNAME/ES_PASSWORD) and construct an authenticated # ES_SERVER URL; tracing would leak them into the CI logs. + +# Self-managed HyperShift. Both files are written by the hypershift-etcd-perf +# chain and absent everywhere else, so this is a no-op on a standalone cluster. +# +# A hosted cluster is split across two clusters: the workload is applied to the +# guest API server, but the etcd and kube-apiserver being measured are pods on +# the management cluster. So the load goes to nested_kubeconfig while the +# metrics come from the endpoint file hypershift-hcp-metrics-endpoint prepared. +HCP_METRICS_ENDPOINT="${SHARED_DIR}/hcp-metrics-endpoint.yml" +HOSTED_CLUSTER=false +if [[ -s "${SHARED_DIR}/nested_kubeconfig" && -s "${HCP_METRICS_ENDPOINT}" ]]; then + HOSTED_CLUSTER=true + echo "Hosted cluster detected; driving the workload against the guest API server" + export KUBECONFIG="${SHARED_DIR}/nested_kubeconfig" +fi + oc projects oc version pushd /tmp @@ -32,7 +48,12 @@ sed -i '/index\.sh/s|WORKLOAD="$WORKLOAD"|WORKLOAD="annotation-churn"|' run.sh EXTRA_FLAGS="--iterations=${ANNOTATION_CHURN_ITERATIONS} --patch-replicas=${PATCH_REPLICAS} --patch-rounds=${PATCH_ROUNDS}" # Flags not handled by run.sh EXTRA_FLAGS+=" --gc-metrics=${GC_METRICS} --profile-type=${PROFILE_TYPE}" -EXTRA_FLAGS+=" --metrics-profile=${METRICS_PROFILES}" +if [[ "${HOSTED_CLUSTER}" == "true" ]]; then + # The endpoint file names its own profiles, so --metrics-profile does not apply. + EXTRA_FLAGS+=" --metrics-endpoint=${HCP_METRICS_ENDPOINT}" +else + EXTRA_FLAGS+=" --metrics-profile=${METRICS_PROFILES}" +fi if [[ "${ENABLE_LOCAL_INDEX}" == "true" ]]; then EXTRA_FLAGS+=" --local-indexing" fi @@ -43,6 +64,17 @@ fi EXTRA_FLAGS+=" ${KB_FLAGS}" export ES_SERVER="https://$ES_USERNAME:$ES_PASSWORD@$ES_HOST" +if [[ "${HOSTED_CLUSTER}" == "true" ]]; then + # run.sh adds --es-server only when it is not already passing a metrics + # endpoint, and the two are mutually exclusive there. Leaving ES_SERVER set + # would have it add both. The endpoint file indexes locally instead, and + # index.sh exits cleanly when ES_SERVER is empty. + # + # Indexing hosted control plane series to Elasticsearch, and so Orion + # regression detection over them, needs run.sh to grow a self-managed + # HyperShift branch next to its existing ROSA and ARO ones. + export ES_SERVER="" +fi export EXTRA_FLAGS UUID ADDITIONAL_PARAMS set +o errexit