diff --git a/ci-operator/step-registry/openshift-qe/etcd-density-crashloop-flood/OWNERS b/ci-operator/step-registry/openshift-qe/etcd-density-crashloop-flood/OWNERS new file mode 120000 index 000000000000..ec405d65a79d --- /dev/null +++ b/ci-operator/step-registry/openshift-qe/etcd-density-crashloop-flood/OWNERS @@ -0,0 +1 @@ +../OWNERS \ No newline at end of file diff --git a/ci-operator/step-registry/openshift-qe/etcd-density-crashloop-flood/openshift-qe-etcd-density-crashloop-flood-commands.sh b/ci-operator/step-registry/openshift-qe/etcd-density-crashloop-flood/openshift-qe-etcd-density-crashloop-flood-commands.sh new file mode 100755 index 000000000000..1e9dc78a65f7 --- /dev/null +++ b/ci-operator/step-registry/openshift-qe/etcd-density-crashloop-flood/openshift-qe-etcd-density-crashloop-flood-commands.sh @@ -0,0 +1,58 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail +# Do not enable `set -x`: these scripts read and export Elasticsearch +# credentials (ES_USERNAME/ES_PASSWORD) and construct an authenticated +# ES_SERVER URL; tracing would leak them into the CI logs. +oc projects +oc version +pushd /tmp + +ES_SECRETS_PATH=${ES_SECRETS_PATH:-/secret} +ES_HOST="search-ocp-qe-perf-scale-test-elk-hcm7wtsqpxy7xogbu72bor4uve.us-east-1.es.amazonaws.com" +ES_PASSWORD=$(cat "${ES_SECRETS_PATH}/password") +ES_USERNAME=$(cat "${ES_SECRETS_PATH}/username") +if [ -e "${ES_SECRETS_PATH}/host" ]; then + ES_HOST=$(cat "${ES_SECRETS_PATH}/host") +fi +UUID=$(uuidgen) + +REPO_URL="https://github.com/cloud-bulldozer/e2e-benchmarking" +LATEST_TAG=$(git ls-remote --tags "${REPO_URL}.git" | awk -F'refs/tags/' '{print $2}' | grep -v '\^{}' | sort -V | tail -n1) +TAG_OPTION="--branch $(if [ "$E2E_VERSION" == "default" ]; then echo "$LATEST_TAG"; else echo "$E2E_VERSION"; fi)" +git clone $REPO_URL $TAG_OPTION --depth 1 +pushd e2e-benchmarking/workloads/kube-burner-ocp-wrapper + +export WORKLOAD="etcd-density crashloop-flood" +# Patch the indexing call in run.sh so the fingerprint records "crashloop-flood" +sed -i '/index\.sh/s|WORKLOAD="$WORKLOAD"|WORKLOAD="crashloop-flood"|' run.sh + +# Variant-specific flags +EXTRA_FLAGS="--iterations=${CRASHLOOP_FLOOD_ITERATIONS} --crashloop-replicas=${CRASHLOOP_REPLICAS} --soak-duration=${SOAK_DURATION}" +# Flags not handled by run.sh +EXTRA_FLAGS+=" --gc-metrics=${GC_METRICS} --profile-type=${PROFILE_TYPE}" +EXTRA_FLAGS+=" --metrics-profile=${METRICS_PROFILES}" +if [[ "${ENABLE_LOCAL_INDEX}" == "true" ]]; then + EXTRA_FLAGS+=" --local-indexing" +fi +if [[ -n "${USER_METADATA}" ]]; then + echo "${USER_METADATA}" > user-metadata.yaml + EXTRA_FLAGS+=" --user-metadata=user-metadata.yaml" +fi +EXTRA_FLAGS+=" ${KB_FLAGS}" + +export ES_SERVER="https://$ES_USERNAME:$ES_PASSWORD@$ES_HOST" +export EXTRA_FLAGS UUID ADDITIONAL_PARAMS + +set +o errexit +./run.sh +RUN_EXIT_CODE=$? +set -o errexit + +METRICS_FOLDER="collected-metrics-${UUID}" +if [[ -d ${METRICS_FOLDER} ]]; then + cp -r ${METRICS_FOLDER} "${ARTIFACT_DIR}/" +fi + +exit ${RUN_EXIT_CODE} diff --git a/ci-operator/step-registry/openshift-qe/etcd-density-crashloop-flood/openshift-qe-etcd-density-crashloop-flood-ref.metadata.json b/ci-operator/step-registry/openshift-qe/etcd-density-crashloop-flood/openshift-qe-etcd-density-crashloop-flood-ref.metadata.json new file mode 100644 index 000000000000..afd4c30f478c --- /dev/null +++ b/ci-operator/step-registry/openshift-qe/etcd-density-crashloop-flood/openshift-qe-etcd-density-crashloop-flood-ref.metadata.json @@ -0,0 +1,11 @@ +{ + "path": "openshift-qe/etcd-density-crashloop-flood/openshift-qe-etcd-density-crashloop-flood-ref.yaml", + "owners": { + "approvers": [ + "perfscale-ocp-approvers" + ], + "reviewers": [ + "perfscale-ocp-reviewers" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/openshift-qe/etcd-density-crashloop-flood/openshift-qe-etcd-density-crashloop-flood-ref.yaml b/ci-operator/step-registry/openshift-qe/etcd-density-crashloop-flood/openshift-qe-etcd-density-crashloop-flood-ref.yaml new file mode 100644 index 000000000000..fd25b9d7fc25 --- /dev/null +++ b/ci-operator/step-registry/openshift-qe/etcd-density-crashloop-flood/openshift-qe-etcd-density-crashloop-flood-ref.yaml @@ -0,0 +1,98 @@ +ref: + as: openshift-qe-etcd-density-crashloop-flood + from_image: + namespace: ci + name: ocp-qe-perfscale-ci + tag: latest + cli: latest + env: + - name: E2E_VERSION + default: "default" + documentation: |- + Override the e2e version + - name: KUBE_BURNER_VERSION + default: "default" + documentation: |- + Override the kube burner version + - name: CRASHLOOP_FLOOD_ITERATIONS + default: "20" + documentation: |- + Number of namespaces to create crashlooping pods in. Prefixed so it only + applies to this step and does not collide with the ITERATIONS consumed by + other kube-burner workloads (e.g. crd-scale) that may be co-scheduled in + the same job. + - name: CRASHLOOP_REPLICAS + default: "100" + documentation: |- + Crashlooping pods per iteration. Each pod runs "exit 1" with + restartPolicy: Always, so it emits roughly 6 events per crash cycle + (Scheduled, Pulling, Pulled, Created, Started, BackOff) continuously. + - name: SOAK_DURATION + default: "30m" + documentation: |- + How long to let events accumulate after the pods are created. This is the + measurement window: the pods are only there to keep producing events. + - name: KB_FLAGS + default: "" + documentation: |- + Additional flags for the kube-burner command. Reused across other kube-burner based workloads + - name: GC + default: "true" + documentation: |- + Default is true, which means clean up the pod/resource that kube-burner ocp created, you can set it to false to keep the resource + - name: GC_METRICS + default: "false" + documentation: |- + Default is false, which means skip collecting metrics, you can set it to true to collect metrics. + - name: PROFILE_TYPE + default: "regular" + documentation: |- + Metrics profile type: regular, reporting, or both + - name: METRICS_PROFILES + default: "metrics.yml,etcd-density-metrics.yml" + documentation: |- + Comma separated list of metrics profiles to use + - name: ENABLE_LOCAL_INDEX + default: "false" + documentation: |- + Trigger to enable local indexing + - name: ES_SECRETS_PATH + default: "" + documentation: |- + Override elasticsearch secrets path. + - name: USER_METADATA + default: "" + documentation: |- + Arbitrary metadata provided to the kube-burner workload in YAML format + - name: ADDITIONAL_PARAMS + default: '{}' + documentation: |- + JSON value to upload additonal metadata to the fingerprint + commands: openshift-qe-etcd-density-crashloop-flood-commands.sh + timeout: 13h + credentials: + - collection: ocp-perfscale-cpt + group: ocp-qe-perfscale-es-creds + mount_path: /secret + - collection: stackrox-perfscale-elasticsearch + group: credentials + mount_path: /secret_stackrox + resources: + requests: + cpu: 100m + memory: 100Mi + documentation: >- + This step runs the etcd-density crashloop-flood workload to stress etcd with + realistic event pressure from crashlooping pods. + + + Unlike event-storm, which creates synthetic Event objects, this deploys pods + that actually crash and so generates events from kubelet, scheduler and + controller sources at once. That exercises the count-increment (PATCH) path + for keys that already exist, a different etcd mvcc profile from creates, and + the shape most real-world event storms take. + + + Defaults are kube-burner-ocp's own. The OpenShift Performance and Scale team + characterised standard against sharded etcd with + CRASHLOOP_FLOOD_ITERATIONS=20 and SOAK_DURATION=10m on a 24 worker cluster.