From e026c100e1f703278ca4218f5b1b4f72a35877d5 Mon Sep 17 00:00:00 2001 From: mehabhalodiya Date: Mon, 26 Oct 2026 17:29:25 +0000 Subject: [PATCH] Add etcd-density event-storm step Adds openshift-qe-etcd-density-event-storm alongside the existing annotation-churn and db-quota-pressure steps, completing the set of etcd-density variants the OpenShift Performance and Scale team characterised standard against sharded etcd with. event-storm floods the cluster with synthetic Event objects in the background while a concurrent victim workload runs, so what is measured is the impact of the flood on ordinary cluster operations rather than the flood itself. It is the create-heavy counterpart to crashloop-flood, which drives comparable pressure through the PATCH path. Defaults are kube-burner-ocp's own; the characterisation values are recorded in the step documentation rather than baked in, so a job config chooses them. Co-Authored-By: Claude Opus 5 --- .../etcd-density-event-storm/OWNERS | 1 + ...ft-qe-etcd-density-event-storm-commands.sh | 61 ++++++++++ ...etcd-density-event-storm-ref.metadata.json | 11 ++ ...shift-qe-etcd-density-event-storm-ref.yaml | 114 ++++++++++++++++++ 4 files changed, 187 insertions(+) create mode 120000 ci-operator/step-registry/openshift-qe/etcd-density-event-storm/OWNERS create mode 100755 ci-operator/step-registry/openshift-qe/etcd-density-event-storm/openshift-qe-etcd-density-event-storm-commands.sh create mode 100644 ci-operator/step-registry/openshift-qe/etcd-density-event-storm/openshift-qe-etcd-density-event-storm-ref.metadata.json create mode 100644 ci-operator/step-registry/openshift-qe/etcd-density-event-storm/openshift-qe-etcd-density-event-storm-ref.yaml diff --git a/ci-operator/step-registry/openshift-qe/etcd-density-event-storm/OWNERS b/ci-operator/step-registry/openshift-qe/etcd-density-event-storm/OWNERS new file mode 120000 index 000000000000..ec405d65a79d --- /dev/null +++ b/ci-operator/step-registry/openshift-qe/etcd-density-event-storm/OWNERS @@ -0,0 +1 @@ +../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/openshift-qe/etcd-density-event-storm/openshift-qe-etcd-density-event-storm-commands.sh b/ci-operator/step-registry/openshift-qe/etcd-density-event-storm/openshift-qe-etcd-density-event-storm-commands.sh new file mode 100755 index 000000000000..a3ff7d5e022f --- /dev/null +++ b/ci-operator/step-registry/openshift-qe/etcd-density-event-storm/openshift-qe-etcd-density-event-storm-commands.sh @@ -0,0 +1,61 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail +# Do not enable `set -x`: these scripts read and export Elasticsearch +# credentials (ES_USERNAME/ES_PASSWORD) and construct an authenticated +# ES_SERVER URL; tracing would leak them into the CI logs. +oc projects +oc version +pushd /tmp + +ES_SECRETS_PATH=${ES_SECRETS_PATH:-/secret} +ES_HOST="search-ocp-qe-perf-scale-test-elk-hcm7wtsqpxy7xogbu72bor4uve.us-east-1.es.amazonaws.com" +ES_PASSWORD=$(cat "${ES_SECRETS_PATH}/password") +ES_USERNAME=$(cat "${ES_SECRETS_PATH}/username") +if [ -e "${ES_SECRETS_PATH}/host" ]; then + ES_HOST=$(cat "${ES_SECRETS_PATH}/host") +fi +UUID=$(uuidgen) + +REPO_URL="https://github.com/cloud-bulldozer/e2e-benchmarking" +LATEST_TAG=$(git ls-remote --tags "${REPO_URL}.git" | awk -F'refs/tags/' '{print $2}' | grep -v '\^{}' | sort -V | tail -n1) +TAG_OPTION="--branch $(if [ "$E2E_VERSION" == "default" ]; then echo "$LATEST_TAG"; else echo "$E2E_VERSION"; fi)" +git clone $REPO_URL $TAG_OPTION --depth 1 +pushd e2e-benchmarking/workloads/kube-burner-ocp-wrapper + +export WORKLOAD="etcd-density event-storm" +# Patch the indexing call in run.sh so the fingerprint records "event-storm" +sed -i '/index\.sh/s|WORKLOAD="$WORKLOAD"|WORKLOAD="event-storm"|' run.sh + +# Variant-specific flags. --iterations sizes the victim workload, the event-* +# and storm-* flags size the background storm it has to survive. +EXTRA_FLAGS="--iterations=${EVENT_STORM_ITERATIONS} --pod-replicas=${EVENT_STORM_POD_REPLICAS}" +EXTRA_FLAGS+=" --event-iterations=${EVENT_ITERATIONS} --event-replicas=${EVENT_REPLICAS}" +EXTRA_FLAGS+=" --storm-workers=${STORM_WORKERS} --storm-delay=${STORM_DELAY}" +# Flags not handled by run.sh +EXTRA_FLAGS+=" --gc-metrics=${GC_METRICS} --profile-type=${PROFILE_TYPE}" +EXTRA_FLAGS+=" --metrics-profile=${METRICS_PROFILES}" +if [[ "${ENABLE_LOCAL_INDEX}" == "true" ]]; then + EXTRA_FLAGS+=" --local-indexing" +fi +if [[ -n "${USER_METADATA}" ]]; then + echo "${USER_METADATA}" > user-metadata.yaml + EXTRA_FLAGS+=" --user-metadata=user-metadata.yaml" +fi +EXTRA_FLAGS+=" ${KB_FLAGS}" + +export ES_SERVER="https://$ES_USERNAME:$ES_PASSWORD@$ES_HOST" +export EXTRA_FLAGS UUID ADDITIONAL_PARAMS + +set +o errexit +./run.sh +RUN_EXIT_CODE=$? +set -o errexit + +METRICS_FOLDER="collected-metrics-${UUID}" +if [[ -d ${METRICS_FOLDER} ]]; then + cp -r ${METRICS_FOLDER} "${ARTIFACT_DIR}/" +fi + +exit ${RUN_EXIT_CODE} diff --git a/ci-operator/step-registry/openshift-qe/etcd-density-event-storm/openshift-qe-etcd-density-event-storm-ref.metadata.json b/ci-operator/step-registry/openshift-qe/etcd-density-event-storm/openshift-qe-etcd-density-event-storm-ref.metadata.json new file mode 100644 index 000000000000..d729abbb2737 --- /dev/null +++ b/ci-operator/step-registry/openshift-qe/etcd-density-event-storm/openshift-qe-etcd-density-event-storm-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "openshift-qe/etcd-density-event-storm/openshift-qe-etcd-density-event-storm-ref.yaml", + "owners": { + "approvers": [ + "perfscale-ocp-approvers" + ], + "reviewers": [ + "perfscale-ocp-reviewers" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/openshift-qe/etcd-density-event-storm/openshift-qe-etcd-density-event-storm-ref.yaml b/ci-operator/step-registry/openshift-qe/etcd-density-event-storm/openshift-qe-etcd-density-event-storm-ref.yaml new file mode 100644 index 000000000000..3d9ab4fec316 --- /dev/null +++ b/ci-operator/step-registry/openshift-qe/etcd-density-event-storm/openshift-qe-etcd-density-event-storm-ref.yaml @@ -0,0 +1,114 @@ +ref: + as: openshift-qe-etcd-density-event-storm + from_image: + namespace: ci + name: ocp-qe-perfscale-ci + tag: latest + cli: latest + env: + - name: E2E_VERSION + default: "default" + documentation: |- + Override the e2e version + - name: KUBE_BURNER_VERSION + default: "default" + documentation: |- + Override the kube burner version + - name: EVENT_STORM_ITERATIONS + default: "100" + documentation: |- + Victim workload iterations. The victim workload is what measures the + impact of the storm, so this sizes the measurement, not the storm. + Prefixed so it only applies to this step and does not collide with the + ITERATIONS consumed by other kube-burner workloads (e.g. crd-scale) that + may be co-scheduled in the same job. + - name: EVENT_STORM_POD_REPLICAS + default: "3" + documentation: |- + Pod replicas per victim deployment. Prefixed for the same reason as + EVENT_STORM_ITERATIONS. + - name: EVENT_ITERATIONS + default: "2000" + documentation: |- + Event creation iterations for the background storm. Total synthetic + events is roughly EVENT_ITERATIONS * EVENT_REPLICAS. + - name: EVENT_REPLICAS + default: "100" + documentation: |- + Event replicas per iteration. Each synthetic Event is about 1 KiB. + - name: STORM_WORKERS + default: "20" + documentation: |- + Parallel workers creating the storm, via kube-burner's background hook. + - name: STORM_DELAY + default: "15" + documentation: |- + Seconds to let the storm build pressure before the victim workload + starts, so the victim is created into an already flooded cluster rather + than racing the flood. + - name: KB_FLAGS + default: "" + documentation: |- + Additional flags for the kube-burner command. Reused across other kube-burner based workloads + - name: GC + default: "true" + documentation: |- + Default is true, which means clean up the pod/resource that kube-burner ocp created, you can set it to false to keep the resource + - name: GC_METRICS + default: "false" + documentation: |- + Default is false, which means skip collecting metrics, you can set it to true to collect metrics. + - name: PROFILE_TYPE + default: "regular" + documentation: |- + Metrics profile type: regular, reporting, or both + - name: METRICS_PROFILES + default: "metrics.yml,etcd-density-metrics.yml" + documentation: |- + Comma separated list of metrics profiles to use + - name: ENABLE_LOCAL_INDEX + default: "false" + documentation: |- + Trigger to enable local indexing + - name: ES_SECRETS_PATH + default: "" + documentation: |- + Override elasticsearch secrets path. + - name: USER_METADATA + default: "" + documentation: |- + Arbitrary metadata provided to the kube-burner workload in YAML format + - name: ADDITIONAL_PARAMS + default: '{}' + documentation: |- + JSON value to upload additonal metadata to the fingerprint + commands: openshift-qe-etcd-density-event-storm-commands.sh + timeout: 13h + credentials: + - collection: ocp-perfscale-cpt + group: ocp-qe-perfscale-es-creds + mount_path: /secret + - collection: stackrox-perfscale-elasticsearch + group: credentials + mount_path: /secret_stackrox + resources: + requests: + cpu: 100m + memory: 100Mi + documentation: >- + This step runs the etcd-density event-storm workload to stress etcd with a + sustained flood of synthetic Event objects. + + + A background job creates the storm while a concurrent victim workload runs + against the same cluster, so what is measured is the impact of the flood on + ordinary cluster operations rather than the flood itself. This is the + create-heavy counterpart to crashloop-flood, which drives the same pressure + through the PATCH path instead. + + + Defaults are kube-burner-ocp's own. The OpenShift Performance and Scale + team characterised standard against sharded etcd with + EVENT_STORM_ITERATIONS=900 and EVENT_STORM_POD_REPLICAS=2, giving 200K + synthetic events against a victim workload of 900 iterations, on a 24 + worker cluster.