From 0e5636893f53396c021b89f6494c624b79a430b1 Mon Sep 17 00:00:00 2001 From: Dheeraj Date: Thu, 4 Jun 2026 18:55:40 +0530 Subject: [PATCH 1/3] feat(EC-1816): add multi-component stress benchmark Add a stress benchmark under benchmark/stress/ that validates a multi-component snapshot with configurable worker count, simulating real-world release pipeline workloads that caused OOM (EC-1805). - Component count controlled via EC_STRESS_COMPONENTS (default 10) - Worker count controlled via EC_STRESS_WORKERS (default 35) - Uses the same golden-container image as the simple benchmark, duplicated across components at runtime - Reuses the existing benchmark/internal/suite harness - Includes prepare_data.sh to regenerate offline data archive - Automatically supported by make benchmark_stress via Makefile wildcard rules Resolves: EC-1816 Co-Authored-By: Claude Opus 4.6 --- benchmark/stress/prepare_data.sh | 48 +++++++++ benchmark/stress/stress.go | 175 +++++++++++++++++++++++++++++++ 2 files changed, 223 insertions(+) create mode 100755 benchmark/stress/prepare_data.sh create mode 100644 benchmark/stress/stress.go diff --git a/benchmark/stress/prepare_data.sh b/benchmark/stress/prepare_data.sh new file mode 100755 index 000000000..01d19d095 --- /dev/null +++ b/benchmark/stress/prepare_data.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Copyright The Conforma Contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +# Creates the files in the data directory that should contain all the data +# needed to run the benchmark, uses the ../offliner for images and plain git +# clone for the git data dependency. Uses the same golden-container image as the +# simple benchmark -- the stress benchmark duplicates it across components at +# runtime. +set -o errexit +set -o nounset +set -o pipefail + +offliner="$(git rev-parse --show-toplevel)/benchmark/offliner" + +dir="$(mktemp -d)" +trap 'rm -rf "${dir}"' EXIT + +( + cd "${dir}" + + imgs=( + quay.io/redhat-user-workloads/rhtap-contract-tenant/golden-container/golden-container@sha256:166e38c156fa81d577a7ba7a948b68c79005a06e302779d1bebc7d31e8bea315 + quay.io/konflux-ci/tekton-catalog/data-acceptable-bundles@sha256:1e70b8f672388838f20a7d45e145e31e99dab06cefa1c5514d6ce41c8bbea1b0 + quay.io/enterprise-contract/ec-release-policy@sha256:64617f0c45689ef7152c5cfbd4cd5709a3126e4ab7482eb6acd994387fe2d4ba + ) + + for img in "${imgs[@]}"; do + go run -C "${offliner}" . "${img}" "${dir}/data/registry/data" + done + + git clone --no-checkout https://github.com/release-engineering/rhtap-ec-policy.git data/git/rhtap-ec-policy.git +) + +tar czf data.tar.gz -C "${dir}" . diff --git a/benchmark/stress/stress.go b/benchmark/stress/stress.go new file mode 100644 index 000000000..d246b6424 --- /dev/null +++ b/benchmark/stress/stress.go @@ -0,0 +1,175 @@ +// Copyright The Conforma Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +// Stress benchmark validating a multi-component snapshot with a configurable +// number of workers, simulating real-world release pipeline workloads. The +// component count and worker count are controlled via the EC_STRESS_COMPONENTS +// and EC_STRESS_WORKERS environment variables respectively. Uses the same +// golden-container image data as the simple benchmark, duplicated across +// components to create memory pressure. The prepare_data.sh script can be used +// to re-populate the data directory. +package main + +import ( + "encoding/json" + "fmt" + "os" + "path" + "strconv" + + "golang.org/x/benchmarks/driver" + + "github.com/conforma/cli/benchmark/internal/registry" + "github.com/conforma/cli/benchmark/internal/suite" + "github.com/conforma/cli/benchmark/internal/untar" +) + +const ( + defaultComponents = 10 + defaultWorkers = 35 +) + +func main() { + driver.Main("Stress", benchmark) +} + +func envInt(name string, fallback int) int { + v, ok := os.LookupEnv(name) + if !ok { + return fallback + } + n, err := strconv.Atoi(v) + if err != nil { + panic(fmt.Sprintf("invalid %s value %q: %v", name, v, err)) + } + return n +} + +func setup() (string, suite.Closer) { + dir, err := untar.UnTar("data.tar.gz") + if err != nil { + panic(err) + } + + closer, err := registry.Launch(path.Join(dir, "data/registry/data")) + if err != nil { + panic(err) + } + + return dir, func() { + closer() + os.RemoveAll(dir) + } +} + +type component struct { + Name string `json:"name"` + ContainerImage string `json:"containerImage"` + Source *source `json:"source,omitempty"` +} + +type source struct { + Git gitSource `json:"git"` +} + +type gitSource struct { + URL string `json:"url"` + Revision string `json:"revision"` +} + +type snapshot struct { + Components []component `json:"components"` +} + +func buildSnapshot(n int) string { + s := snapshot{Components: make([]component, n)} + for i := range s.Components { + s.Components[i] = component{ + Name: fmt.Sprintf("golden-container-%d", i), + ContainerImage: "quay.io/redhat-user-workloads/rhtap-contract-tenant/golden-container/golden-container@sha256:166e38c156fa81d577a7ba7a948b68c79005a06e302779d1bebc7d31e8bea315", + Source: &source{ + Git: gitSource{ + URL: "https://github.com/enterprise-contract/golden-container.git", + Revision: "8327c1ce7472b017b9396fe26d5d5e1ed0eb61cc", + }, + }, + } + } + + data, err := json.Marshal(s) + if err != nil { + panic(err) + } + return string(data) +} + +func benchmark() driver.Result { + dir, closer := setup() + defer closer() + + components := envInt("EC_STRESS_COMPONENTS", defaultComponents) + workers := envInt("EC_STRESS_WORKERS", defaultWorkers) + + return driver.Benchmark(run(dir, components, workers)) +} + +func ec(dir string, components, workers int) func() { + snap := buildSnapshot(components) + + policy := fmt.Sprintf(`{ +"publicKey": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEZP/0htjhVt2y0ohjgtIIgICOtQtA\nnaYJRuLprwIv6FDhZ5yFjYUEtsmoNcW7rx2KM6FOXGsCX3BNc7qhHELT+g==\n-----END PUBLIC KEY-----", +"sources": [ +{ + "data": [ + "git::file://%s/data/git/rhtap-ec-policy.git//data?ref=a524ee2f2f7774f6f360eb64c4cb24004de52aae", + "oci::quay.io/konflux-ci/tekton-catalog/data-acceptable-bundles@sha256:1e70b8f672388838f20a7d45e145e31e99dab06cefa1c5514d6ce41c8bbea1b0" + ], + "policy": [ + "oci::quay.io/enterprise-contract/ec-release-policy@sha256:64617f0c45689ef7152c5cfbd4cd5709a3126e4ab7482eb6acd994387fe2d4ba" + ], + "config": { + "include": [ + "@redhat" + ] + } +} +] +}`, dir) + + return func() { + if err := suite.Execute([]string{ + "validate", + "image", + "--json-input", + snap, + "--policy", + policy, + "--ignore-rekor", + "--workers", + strconv.Itoa(workers), + "--effective-time", + "2024-12-10T00:00:00Z", + }); err != nil { + panic(err) + } + } +} + +func run(dir string, components, workers int) func(n uint64) { + return func(n uint64) { + driver.Parallel(n, 1, ec(dir, components, workers)) + } +} From 1e10a9614a24d8a45aa54b52d67438e521c7f58e Mon Sep 17 00:00:00 2001 From: Dheeraj Date: Tue, 16 Jun 2026 17:52:34 +0530 Subject: [PATCH 2/3] feat(EC-1816): host stress benchmark data on Quay Pull pre-built data.tar.gz from quay.io/conforma/benchmark-data in prepare_data.sh, falling back to upstream regeneration. Add push_data.sh for uploading the archive. Resolves: EC-1816 Co-Authored-By: Claude Opus 4.6 --- benchmark/stress/prepare_data.sh | 18 ++++++++++++++---- benchmark/stress/push_data.sh | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 4 deletions(-) create mode 100755 benchmark/stress/push_data.sh diff --git a/benchmark/stress/prepare_data.sh b/benchmark/stress/prepare_data.sh index 01d19d095..37d605aba 100755 --- a/benchmark/stress/prepare_data.sh +++ b/benchmark/stress/prepare_data.sh @@ -16,14 +16,24 @@ # SPDX-License-Identifier: Apache-2.0 # Creates the files in the data directory that should contain all the data -# needed to run the benchmark, uses the ../offliner for images and plain git -# clone for the git data dependency. Uses the same golden-container image as the -# simple benchmark -- the stress benchmark duplicates it across components at -# runtime. +# needed to run the benchmark. Tries to pull a pre-built data archive from Quay +# first, falling back to building from upstream using the ../offliner for images +# and plain git clone for the git data dependency. Uses the same +# golden-container image as the simple benchmark -- the stress benchmark +# duplicates it across components at runtime. set -o errexit set -o nounset set -o pipefail +quay_ref="quay.io/conforma/benchmark-data:stress-v1" + +if command -v oras &>/dev/null && oras pull "${quay_ref}" -o . 2>/dev/null; then + echo "Downloaded data.tar.gz from ${quay_ref}" + exit 0 +fi + +echo "Quay pull failed or oras not available, regenerating from upstream..." + offliner="$(git rev-parse --show-toplevel)/benchmark/offliner" dir="$(mktemp -d)" diff --git a/benchmark/stress/push_data.sh b/benchmark/stress/push_data.sh new file mode 100755 index 000000000..9f5feee1c --- /dev/null +++ b/benchmark/stress/push_data.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright The Conforma Contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +# Pushes the local data.tar.gz to Quay as an OCI artifact. Requires prior +# authentication via `oras login quay.io`. +set -o errexit +set -o nounset +set -o pipefail + +quay_ref="quay.io/conforma/benchmark-data:stress-v1" + +if [[ ! -f data.tar.gz ]]; then + echo "data.tar.gz not found, run prepare_data.sh first" >&2 + exit 1 +fi + +oras push "${quay_ref}" data.tar.gz +echo "Pushed data.tar.gz to ${quay_ref}" From aa42a5af020d97b1d3e4490df23ff6437152457d Mon Sep 17 00:00:00 2001 From: Dheeraj Date: Tue, 16 Jun 2026 18:00:45 +0530 Subject: [PATCH 3/3] fix(EC-1816): validate envInt returns positive value Reject zero and negative values for EC_STRESS_COMPONENTS and EC_STRESS_WORKERS to fail fast instead of producing meaningless benchmark results. Co-Authored-By: Claude Opus 4.6 --- benchmark/stress/stress.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmark/stress/stress.go b/benchmark/stress/stress.go index d246b6424..179a9f839 100644 --- a/benchmark/stress/stress.go +++ b/benchmark/stress/stress.go @@ -55,6 +55,9 @@ func envInt(name string, fallback int) int { if err != nil { panic(fmt.Sprintf("invalid %s value %q: %v", name, v, err)) } + if n < 1 { + panic(fmt.Sprintf("%s must be >= 1, got %d", name, n)) + } return n }