diff --git a/.dockerignore b/.dockerignore index 0ad8e3917d2..1e5b618210f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,6 +6,7 @@ * !src !deps +!utils/build !utils/build.config !utils/certs !utils/ci diff --git a/Jenkinsfile b/Jenkinsfile index c5ace14abd7..94224b67629 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -832,29 +832,34 @@ pipeline { ' --target build-ci' + ' --build-arg REPOS="' + prRepos() + '"' + ' --build-arg POINT_RELEASE=.7' + - " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}" + - " --build-arg DAOS_DEPS_INSTALL=yes" + " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}" } } steps { script { + sh label: 'Collect dependency RPMs built into the image', + script: 'mkdir -p rpms/deps && ' + + 'cp /home/daos/rpms/deps/*.rpm rpms/deps/ 2>/dev/null || true' job_step_update( sconsBuild(parallel_build: true, stash_files: 'ci/test_files_to_stash.txt', build_deps: 'no', stash_opt: true, + scons_exe: 'utils/build/build_daos.sh', scons_args: sconsArgs() + - ' PREFIX=/opt/daos TARGET_TYPE=release')) - sh label: 'Generate RPMs', - script: './ci/rpm/gen_rpms.sh el9 "' + env.DAOS_RELVAL + '"' + ' TARGET_TYPE=release')) + sh label: 'Build DAOS RPMs', + script: 'DISTRO=el9 DAOS_RELVAL="' + env.DAOS_RELVAL + + '" utils/build/build_packages.sh daos' // Go binaries need to be instrumented in order to work reliably // with valgrind. We do this in a separate build because we don't // want to ship the instrumented binaries. job_step_update( sconsBuild(parallel_build: true, build_deps: 'no', + scons_exe: 'utils/build/build_daos.sh', scons_args: sconsArgs() + - ' BUILD_GO_VALGRIND=1 PREFIX=/opt/daos TARGET_TYPE=release')) + ' BUILD_GO_VALGRIND=1 TARGET_TYPE=release')) sh label: 'Stash valgrind install tree for NLT', script: 'tar -C / -cf opt-daos-valgrind.tar opt/daos' stash(name: 'opt-daos-valgrind', includes: 'opt-daos-valgrind.tar') @@ -887,26 +892,30 @@ pipeline { filename 'utils/docker/Dockerfile.leap.15' label 'docker_runner' additionalBuildArgs dockerBuildArgs(repo_type: 'stable', - parallel_build: true, - deps_build: true) + + deps_build: true, + parallel_build: true) + " -t ${sanitized_JOB_NAME()}-leap15" + ' --target build-ci' + ' --build-arg POINT_RELEASE=.6' + - " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}" + - " --build-arg DAOS_DEPS_INSTALL=yes" + " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}" } } steps { script { + sh label: 'Collect dependency RPMs built into the image', + script: 'mkdir -p rpms/deps && ' + + 'cp /home/daos/rpms/deps/*.rpm rpms/deps/ 2>/dev/null || true' job_step_update( sconsBuild(parallel_build: true, stash_files: 'ci/test_files_to_stash.txt', build_deps: 'no', stash_opt: true, + scons_exe: 'utils/build/build_daos.sh', scons_args: sconsArgs() + - ' PREFIX=/opt/daos TARGET_TYPE=release')) - sh label: 'Generate RPMs', - script: './ci/rpm/gen_rpms.sh suse.lp156 "' + env.DAOS_RELVAL + '"' + ' TARGET_TYPE=release')) + sh label: 'Build DAOS RPMs', + script: 'DISTRO=suse.lp156 DAOS_RELVAL="' + env.DAOS_RELVAL + + '" utils/build/build_packages.sh daos' } } post { diff --git a/ci/rpm/build_deps.sh b/ci/rpm/build_deps.sh deleted file mode 100755 index 6ea52e6a597..00000000000 --- a/ci/rpm/build_deps.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -set -euo pipefail - -DAOS_TARGET_TYPE="${DAOS_TARGET_TYPE:-release}" -DAOS_DEPS_JOBS="${DAOS_DEPS_JOBS:-32}" -echo "Building DAOS dependencies... with $DAOS_DEPS_JOBS jobs and target type \"$DAOS_TARGET_TYPE\"" - -scons install --build-deps=only USE_INSTALLED=all PREFIX=/opt/daos \ - TARGET_TYPE="$DAOS_TARGET_TYPE" -j "$DAOS_DEPS_JOBS" diff --git a/ci/rpm/build_success.sh b/ci/rpm/build_success.sh index 93e82ee72c6..51af2a3dcae 100755 --- a/ci/rpm/build_success.sh +++ b/ci/rpm/build_success.sh @@ -11,19 +11,12 @@ if [ -e "${ci_envs}" ]; then source "${ci_envs}" fi -: "${TARGET:=centos9}" - +: "${TARGET:=el9}" artdir="${PWD}/artifacts/${TARGET}" +rpmdir="${PWD}/rpms" rm -rf "$artdir" mkdir -p "$artdir" -mkdir -p "$artdir/daos" -if [ -d /home/daos/rpms/ ]; then - if [ -d /home/daos/rpms/deps ]; then - mkdir -p "$artdir/deps" - cp /home/daos/rpms/deps/*.rpm "${artdir}/deps" - fi - cp /home/daos/rpms/daos/*.rpm "${artdir}/daos" +if [ -d "${rpmdir}" ]; then + cp -r "${rpmdir}"/* "${artdir}" fi - -createrepo "$artdir" diff --git a/ci/rpm/gen_rpms.sh b/ci/rpm/gen_rpms.sh deleted file mode 100755 index 30a4e248952..00000000000 --- a/ci/rpm/gen_rpms.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -# Script for building DAOS RPMs from a DAOS build -set -uex - -id -if [ "$(id -u)" = "0" ]; then - echo "Should not be run as root" - exit 1 -fi - -mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -ci_envs="$mydir/../parse_ci_envs.sh" -if [ -e "${ci_envs}" ]; then - # shellcheck source=parse_ci_envs.sh disable=SC1091 - source "${ci_envs}" -fi - -env - -pushd "${mydir}/../.." || exit 1 -export DISTRO="${1}" -export DAOS_RELVAL="${2}" -rm -f ./*.rpm -rm -rf /home/daos/rpms/* -utils/rpms/build_packages.sh deps -if ls -1 ./*.rpm; then - mkdir -p /home/daos/rpms/deps - cp ./*.rpm /home/daos/rpms/deps - rm -f ./*.rpm -fi -utils/rpms/build_packages.sh daos -mkdir -p /home/daos/rpms/daos -cp ./*.rpm /home/daos/rpms/daos -popd || exit 1 diff --git a/ci/rpm/install_deps.sh b/ci/rpm/install_deps.sh deleted file mode 100755 index 5734a83dd7f..00000000000 --- a/ci/rpm/install_deps.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -# Script for installing DAOS dependencies for the expected version -set -uex - -id -if [ "$(id -u)" = "0" ]; then - echo "Should not be run as root" - exit 1 -fi - -mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" - -env - -pushd "${mydir}/../.." || exit 1 -export DISTRO="${1}" -libfabric_pkg="$(utils/rpms/package_version.sh libfabric dev)" -mercury_pkg="$(utils/rpms/package_version.sh mercury dev)" -argobots_pkg="$(utils/rpms/package_version.sh argobots dev)" -fused_pkg="$(utils/rpms/package_version.sh fused dev)" -isal_pkg="$(utils/rpms/package_version.sh isal dev)" -isal_crypto_pkg="$(utils/rpms/package_version.sh isal_crypto dev)" -daos_spdk_pkg="$(utils/rpms/package_version.sh daos_spdk dev)" -pmdk_pkg="$(utils/rpms/package_version.sh pmdk dev pmemobj)" - -sudo dnf install --allowerasing -y "${libfabric_pkg}" || echo "${libfabric_pkg} not available" -sudo dnf install --allowerasing -y "${mercury_pkg}" || echo "${mercury_pkg} not available" -sudo dnf install --allowerasing -y "${argobots_pkg}" || echo "${argobots_pkg} not available" -sudo dnf install --allowerasing -y "${daos_spdk_pkg}" || echo "${daos_spdk_pkg} not available" -sudo dnf install --allowerasing -y "${fused_pkg}" || echo "${fused_pkg} not available" -sudo dnf install --allowerasing -y "${pmdk_pkg}" || echo "${pmdk_pkg} not available" -sudo dnf install --allowerasing -y "${isal_pkg}" || echo "${isal_pkg} not available" -sudo dnf install --allowerasing -y "${isal_crypto_pkg}" || echo "${isal_crypto_pkg} not available" -popd || exit 1 diff --git a/docs/dev/development.md b/docs/dev/development.md index e7b35f46c0d..fb5ad7ac881 100644 --- a/docs/dev/development.md +++ b/docs/dev/development.md @@ -336,18 +336,18 @@ To build the Docker image directly from GitHub, run the following command: ```bash $ docker build https://github.com/daos-stack/daos.git#master \ - -f utils/docker/Dockerfile.el.8 -t daos + -f utils/docker/Dockerfile.el.9 -t daos ``` or from a local tree: ```bash -$ docker build . -f utils/docker/Dockerfile.el.8 -t daos +$ docker build . -f utils/docker/Dockerfile.el.9 -t daos ``` -This creates a Rocky Linux 8 image, fetches the latest DAOS version from GitHub, +This creates a Rocky Linux 9 image, fetches the latest DAOS version from GitHub, builds it, and installs it in the image. -For Ubuntu and other Linux distributions, replace Dockerfile.el.8 with +For Ubuntu and other Linux distributions, replace Dockerfile.el.9 with Dockerfile.ubuntu or the appropriate version of interest. ### Simple Docker Setup @@ -403,7 +403,7 @@ refer to the next section. The DAOS build process now covers building RPMs for both DAOS and dependencies specified in [`utils/build.config`](../../utils/build.config) (or those that we build regularly with `--build-deps=yes`). The complete list of RPMs is defined -in the [`utils/rpms/build_packages.sh`](../../utils/rpms/build_packages.sh) +in the [`utils/build/build_packages.sh`](../../utils/build/build_packages.sh) script. The RPM (and deb) build process uses [FPM](https://fpm.readthedocs.io/en/latest/getting-started.html). Essentially, it creates rpm packages after a DAOS build. Regardless of how that build is done, @@ -428,3 +428,56 @@ In order to properly upgrade a 3rd party component, do all of the following: 1. Update the `utils/rpms/.changelog` file to document the change and make sure the file is referenced by the `RPM_CHANGELOG=".changelog"` variable in `utils/rpms/.sh`. + +## Unified DAOS Build Procedure + +The scripts under [`utils/build`](../../utils/build) provide a straightforward, +four-step workflow from installing pre-built dependencies to producing the +final RPMs. The same workflow is used by CI, Docker image builds, and bare-host +builds; none of the scripts are Docker-specific. + +For standard builds, the scripts handle the underlying `scons` details. +Advanced users can still invoke `scons` directly or pass additional options +and variables through the `build_*` scripts to produce different binary +variants. + +1. **`install_deps.sh [DISTRO]`** installs pre-built dependency RPMs (e.g. + `argobots-devel`, `mercury-devel`, `libfabric-devel`) matching the versions + expected by the current tree, so that the subsequent build steps can reuse + them instead of rebuilding from source (`USE_INSTALLED=all`). `DISTRO` is + not the OS distro itself, but the standardized RPM naming suffix used by + the DAOS project's own package repos (e.g. `el9`, `suse.lp155`, + `suse.lp156`); if omitted, it's auto-detected from `/etc/os-release`. + Missing packages are reported but do not fail the script, since + dependencies can also be built from source in the next step. Set + `DAOS_DEPS_EXT_REPO` to pull in a custom RPM set published by the DAOS + project (e.g. from [packages.daos.io](https://packages.daos.io/)) as + an extra dnf-format repo, registered only for the duration of the script. + + ```bash + $ DAOS_DEPS_EXT_REPO=https://packages.daos.io/v2.8.0/EL9/packages/x86_64/ \ + utils/build/install_deps.sh el9 + ``` +1. **`build_deps.sh`** builds any dependency not already satisfied by + `install_deps.sh` from source, via `scons install --build-deps=only`. +1. **`build_daos.sh`** builds and installs DAOS itself with scons + (`USE_INSTALLED=all` to reuse the dependencies from the previous steps). +1. **`build_packages.sh [deps|daos|all] [yes|no]`** builds dependency + RPMs, DAOS RPMs, or both with `fpm`. The build type defaults to `all`, + and verification defaults to `yes`. + + RPMs are written to the `deps/` and `daos/` subdirectories under + `RPM_OUTPUT_DIR`, which defaults to `/rpms`. For supported RPM + distributions, the script also generates repository metadata under + `/repodata`, producing a complete RPM repository. + + After each nonempty package group is built, the script runs + `verify_packages.sh`. With the default `yes` setting, validation findings + fail the build. With `no`, verification still runs, but noncritical findings + are reported as warnings. Missing tools, unsupported configurations, and + other critical setup errors remain fatal. `verify_packages.sh` can also be + invoked separately. + + `DISTRO` selects the distribution suffix used in package names, such as + `el9` or `suse.lp156`. It is detected from `/etc/os-release` when unset. + Set `RPM_OUTPUT_DIR` to change the output repository location. \ No newline at end of file diff --git a/utils/build/build_daos.sh b/utils/build/build_daos.sh new file mode 100755 index 00000000000..25bff40dc46 --- /dev/null +++ b/utils/build/build_daos.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# Copyright 2026 Hewlett Packard Enterprise Development LP +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +# shellcheck source=utils/build/build_utils.sh +source "${script_dir}/build_utils.sh" + +usage() { + cat < number of parallel jobs + USE_INSTALLED=all reuse dependencies already installed on the system + PREFIX=/opt/daos installation prefix + +Options: + -c run scons -c command + -h, --help show this help and exit + +Any other argument is forwarded verbatim to scons, e.g.: + + ${0##*/} BUILD_TYPE=debug COMPILER=clang --jobs 8 +EOF +} + +check_help "$@" +check_scons + +jobs_set=false +prefix_set=false +use_installed_set=false +install_set=false +build_deps_set=false +for arg in "$@"; do + case "$arg" in + -c) + scons -c + exit 0 + ;; + -j | --jobs ) + jobs_set=true + ;; + PREFIX=*) + prefix_set=true + ;; + USE_INSTALLED=*) + use_installed_set=true + ;; + install) + install_set=true + ;; + --build-deps=*) + build_deps_set=true + ;; + esac +done + +SCONS_ARGS=() +if ! "$install_set"; then + SCONS_ARGS+=(install) +fi +if ! "$build_deps_set"; then + SCONS_ARGS+=(--build-deps=no ) +fi +if ! "$jobs_set"; then + SCONS_ARGS+=(--jobs "$(nproc)") +fi +if ! "$use_installed_set"; then + SCONS_ARGS+=("USE_INSTALLED=all") +fi +if ! "$prefix_set"; then + SCONS_ARGS+=("PREFIX=/opt/daos") +fi +SCONS_ARGS+=("$@") + +echo "Building DAOS using scons with args:" +echo "${SCONS_ARGS[*]}" + +scons "${SCONS_ARGS[@]}" diff --git a/utils/build/build_deps.sh b/utils/build/build_deps.sh new file mode 100755 index 00000000000..f274662402f --- /dev/null +++ b/utils/build/build_deps.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# Copyright 2025-2026 Hewlett Packard Enterprise Development LP +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +# shellcheck source=utils/build/build_utils.sh +source "${script_dir}/build_utils.sh" + +usage() { + cat < number of parallel jobs + USE_INSTALLED=all reuse dependencies already installed on the system + PREFIX=/opt/daos installation prefix + +Options: + -f, --force wipe \$PREFIX/prereq/release and build/external/release + before building; must be given as the first argument + -h, --help show this help and exit + +Any other argument is forwarded verbatim to scons, e.g.: + + ${0##*/} TARGET_TYPE=debug DEPS=ofi --jobs 8 +EOF +} + +check_help "$@" +check_scons + +force=false +if [ "${1:-}" = "-f" ] || [ "${1:-}" = "--force" ]; then + force=true + shift +fi + +jobs_set=false +prefix_set=false +prefix_value="/opt/daos" +use_installed_set=false + +for arg in "$@"; do + case "$arg" in + -j | --jobs ) + jobs_set=true + ;; + PREFIX=*) + prefix_set=true + prefix_value="${arg#PREFIX=}" + ;; + USE_INSTALLED=*) + use_installed_set=true + ;; + esac +done + +if "$force"; then + echo "Removing \"${prefix_value}/prereq/${BUILD_TYPE:-release}\"" + echo "Removing \"build/external/${BUILD_TYPE:-release}\"" + rm -rf "${prefix_value}/prereq/${BUILD_TYPE:-release}" \ + "build/external/${BUILD_TYPE:-release}" +fi + +SCONS_ARGS=(install --build-deps=only) +if ! "$jobs_set"; then + SCONS_ARGS+=(--jobs "$(nproc)") +fi +if ! "$use_installed_set"; then + SCONS_ARGS+=("USE_INSTALLED=all") +fi +if ! "$prefix_set"; then + SCONS_ARGS+=("PREFIX=${prefix_value}") +fi +SCONS_ARGS+=("$@") + +echo "Building DAOS dependencies using scons with args:" +echo "${SCONS_ARGS[*]}" + +scons "${SCONS_ARGS[@]}" diff --git a/utils/build/build_packages.sh b/utils/build/build_packages.sh new file mode 100755 index 00000000000..3f736c267e0 --- /dev/null +++ b/utils/build/build_packages.sh @@ -0,0 +1,113 @@ +#!/bin/bash +# Copyright 2025 Google LLC +# Copyright 2026 Hewlett Packard Enterprise Development LP +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +set -eEuo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +repo_root="$(cd "${script_dir}/../.." >/dev/null 2>&1 && pwd)" +# shellcheck source=utils/build/build_utils.sh +source "${script_dir}/build_utils.sh" + +usage() { + cat < /dev/null; then + local mode="ERROR" + [ "${verify_rpms}" = "yes" ] || mode="WARNING" + "${script_dir}/verify_packages.sh" "${RPM_OUTPUT_DIR}/${1}" "${DISTRO}" "${mode}" + fi +} + +source utils/sl/setup_local.sh +if [[ "${build_type}" =~ deps|all ]]; then + prepare_rpms_stage deps + utils/rpms/argobots.sh + utils/rpms/fused.sh + utils/rpms/isa-l.sh + utils/rpms/isa-l_crypto.sh + utils/rpms/libfabric.sh + utils/rpms/mercury.sh + utils/rpms/pmdk.sh + utils/rpms/daos-spdk.sh + verify_stage deps +fi + +if [[ "${build_type}" =~ daos|all ]]; then + prepare_rpms_stage daos + utils/rpms/daos.sh + verify_stage daos +fi + +if [[ "${build_type}" =~ deps|daos|all ]] && \ + [[ "${DISTRO}" == el9* || "${DISTRO}" == suse.lp15* ]] && \ + [ "${OUTPUT_TYPE:-rpm}" = "rpm" ]; then + rm -rf "${RPM_OUTPUT_DIR}/repodata" + createrepo "${RPM_OUTPUT_DIR}" +fi diff --git a/utils/build/build_utils.sh b/utils/build/build_utils.sh new file mode 100644 index 00000000000..0ff8a340141 --- /dev/null +++ b/utils/build/build_utils.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Copyright 2026 Hewlett Packard Enterprise Development LP +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# +# Shared helpers sourced by the other utils/build/*.sh scripts. +# Not meant to be executed directly. + +# Scans "$@" for -h/--help and, if found, calls the sourcing script's usage() +# function and exits 0. Must be called after usage() is defined by the caller. +check_help() { + local arg + for arg in "$@"; do + case "${arg}" in + -h | --help) + usage + exit 0 + ;; + esac + done +} + +check_scons() { + [[ -n $(command -v scons 2>/dev/null) ]] || { + echo "ERROR: 'scons' not found on PATH." >&2 + exit 1 + } +} + +# Maps the running OS (from /etc/os-release) to the DISTRO suffix used by +# DAOS RPMs; +detect_distro() { + # shellcheck source=/dev/null + (. /etc/os-release + case "${ID:-}" in + rocky | rhel | centos | almalinux) + echo "el${VERSION_ID%%.*}" + ;; + opensuse-leap | sles) + echo "suse.lp${VERSION_ID//./}" + ;; + *) + echo "ERROR: cannot auto-detect DISTRO for ID=${ID:-unknown}; set it explicitly" >&2 + return 1 + ;; + esac + ) +} \ No newline at end of file diff --git a/utils/build/install_deps.sh b/utils/build/install_deps.sh new file mode 100755 index 00000000000..892c48d1354 --- /dev/null +++ b/utils/build/install_deps.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# +# (C) Copyright 2025 Google LLC +# (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# +# Script for installing DAOS dependencies for the expected version +# +# Usage: +# install_deps.sh [DISTRO] +# +# Args: +# DISTRO Package distro suffix used by DAOS RPMs, e.g. el9, suse.lp155, +# suse.lp156. This is not the OS distro itself, but the +# standardized RPM naming suffix used by the DAOS project's own +# package repos. If omitted, it is auto-detected from +# /etc/os-release. +# +# Env: +# DAOS_DEPS_EXT_REPO Optional dnf-format repo URL with a complete set of +# dependency RPMs, registered as an extra install source +# for the duration of this script only. Requires +# passwordless sudo to write/remove +# /etc/yum.repos.d/daos-deps-extra.repo. +set -uex + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +# shellcheck source=utils/build/build_utils.sh +source "${script_dir}/build_utils.sh" +rpms_dir="$(cd "${script_dir}/../rpms" >/dev/null 2>&1 && pwd)" + +usage() { + cat < /dev/null <<-EOF + [daos-deps-extra] + name=DAOS dependency RPMs extra repo + baseurl=${DAOS_DEPS_EXT_REPO} + enabled=1 + gpgcheck=0 + EOF + trap 'sudo rm -f /etc/yum.repos.d/daos-deps-extra.repo' EXIT +fi + +export DISTRO +libfabric_pkg="$("${rpms_dir}/package_version.sh" libfabric dev)" +mercury_pkg="$("${rpms_dir}/package_version.sh" mercury dev)" +argobots_pkg="$("${rpms_dir}/package_version.sh" argobots dev)" +fused_pkg="$("${rpms_dir}/package_version.sh" fused dev)" +isal_pkg="$("${rpms_dir}/package_version.sh" isal dev)" +isal_crypto_pkg="$("${rpms_dir}/package_version.sh" isal_crypto dev)" +daos_spdk_pkg="$("${rpms_dir}/package_version.sh" daos_spdk dev)" +pmdk_pkg="$("${rpms_dir}/package_version.sh" pmdk dev pmemobj)" + +sudo dnf install --allowerasing -y "${libfabric_pkg}" || echo "${libfabric_pkg} not available" +sudo dnf install --allowerasing -y "${mercury_pkg}" || echo "${mercury_pkg} not available" +sudo dnf install --allowerasing -y "${argobots_pkg}" || echo "${argobots_pkg} not available" +sudo dnf install --allowerasing -y "${daos_spdk_pkg}" || echo "${daos_spdk_pkg} not available" +sudo dnf install --allowerasing -y "${fused_pkg}" || echo "${fused_pkg} not available" +sudo dnf install --allowerasing -y "${pmdk_pkg}" || echo "${pmdk_pkg} not available" +sudo dnf install --allowerasing -y "${isal_pkg}" || echo "${isal_pkg} not available" +sudo dnf install --allowerasing -y "${isal_crypto_pkg}" || echo "${isal_crypto_pkg} not available" diff --git a/utils/rpms/verify_rpms.sh b/utils/build/verify_packages.sh similarity index 80% rename from utils/rpms/verify_rpms.sh rename to utils/build/verify_packages.sh index 3074c778535..e6972c83073 100755 --- a/utils/rpms/verify_rpms.sh +++ b/utils/build/verify_packages.sh @@ -4,9 +4,9 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent # -# verify_rpms.sh +# verify_packages.sh # -# Validate generated RPM packages produced by ci/rpm/gen_rpms.sh. +# Validate generated RPM packages produced by build_packages.sh. # # Checks: # - RPM metadata and payload listing are readable. @@ -31,7 +31,7 @@ # unrelated system devel packages are outside its scope. # # Usage: -# utils/rpms/verify_rpms.sh [RPM_ROOT] [DISTRO] +# utils/build/verify_packages.sh [RPM_ROOT] [DISTRO] [MODE] # # Arguments: # RPM_ROOT Root directory containing either: @@ -40,10 +40,15 @@ # Default: . # DISTRO Distribution identifier used to select the explicit devel to # runtime dependency table. Default: el9. +# MODE ERROR|WARNING. In ERROR mode, validation issues fail the script +# (exit 1). In WARNING mode, issues are reported as warnings and +# the script exits 0. Missing tooling, no RPM files found, or no +# devel dependency table for DISTRO always fail regardless of +# MODE. Default: ERROR. # # Exit codes: -# 0 Validation passed. -# 1 Validation failed or required tooling is missing. +# 0 Validation passed (or MODE=WARNING with no critical errors). +# 1 Validation failed (MODE=ERROR) or required tooling/setup is missing. set -euo pipefail shopt -s nullglob @@ -68,6 +73,16 @@ trap 'rm -rf "${tmpdir}"' EXIT DISTRO="${2:-${DISTRO:-el9}}" +MODE="${3:-ERROR}" +case "${MODE}" in + ERROR|WARNING) + ;; + *) + echo "ERROR: MODE parameter must be 'ERROR' or 'WARNING' (got: ${MODE})" + exit 1 + ;; +esac + declare -i errors=0 declare -A pkg_names declare -A pkg_present @@ -76,7 +91,7 @@ declare -A devel_runtime # Explicit devel to runtime package dependencies. These names are kept # separate by distro because the same library has different RPM names. case "${DISTRO}" in - el*) + el9*) devel_runtime=( [argobots-devel]=argobots [daos-spdk-devel]=daos-spdk @@ -107,6 +122,13 @@ case "${DISTRO}" in exit 1 ;; esac +# Records a validation issue; severity label follows MODE, but the error +# counter is always incremented so the final exit-code decision can use it. +report_error() { + echo "${MODE}: $*" + errors+=1 +} + # Check whether RPM Requires contains a regex pattern. # Args: # $1 path to RPM file @@ -160,14 +182,12 @@ check_rpm_basic() { local pkg_name if ! rpm -K "${rpm_file}" >/dev/null; then - echo "ERROR: rpm verification failed for ${rpm_file}" - errors+=1 + report_error "rpm verification failed for ${rpm_file}" return fi if ! pkg_name="$(rpm -qp --qf '%{NAME}\n' "${rpm_file}")"; then - echo "ERROR: cannot read package name from ${rpm_file}" - errors+=1 + report_error "cannot read package name from ${rpm_file}" return fi @@ -175,8 +195,7 @@ check_rpm_basic() { pkg_present["${pkg_name}"]=1 if ! rpm -qpl "${rpm_file}" >/dev/null; then - echo "ERROR: cannot list payload for ${rpm_file}" - errors+=1 + report_error "cannot list payload for ${rpm_file}" fi } @@ -211,8 +230,7 @@ check_devel_runtime_dependency() { local expected_version if ! expected_version="$(rpm -qp --qf '%{VERSION}-%{RELEASE}\n' "${rpm_file}")"; then - echo "ERROR: cannot determine package version for ${pkg_name} (${rpm_file})" - errors+=1 + report_error "cannot determine package version for ${pkg_name} (${rpm_file})" return fi @@ -224,20 +242,17 @@ check_devel_runtime_dependency() { runtime_pkg="${devel_runtime["${pkg_name}"]:-}" if [ -z "${runtime_pkg}" ]; then - echo "ERROR: no runtime dependency mapping for ${pkg_name} on ${DISTRO}" - errors+=1 + report_error "no runtime dependency mapping for ${pkg_name} on ${DISTRO}" return fi if [ -z "${pkg_present["${runtime_pkg}"]+x}" ]; then - echo "ERROR: ${pkg_name} requires missing runtime package ${runtime_pkg} in generated RPM set" - errors+=1 + report_error "${pkg_name} requires missing runtime package ${runtime_pkg} in generated RPM set" return fi if ! rpm_requires_version_match "${rpm_file}" "${runtime_pkg}" "${expected_version}"; then - echo "ERROR: ${pkg_name} must require ${runtime_pkg} = ${expected_version}" - errors+=1 + report_error "${pkg_name} must require ${runtime_pkg} = ${expected_version}" fi } @@ -271,9 +286,8 @@ check_binary_requires_libc() { if [ "${has_dynamic_elf_binary}" -eq 1 ] && ! rpm_requires_match "${rpm_file}" '^libc\.so\.6'; then - echo "ERROR: ${pkg_name} contains dynamically linked ELF binaries, but RPM" - echo "ERROR: requires metadata is missing libc soname dependency (libc.so.6*)" - errors+=1 + report_error "${pkg_name} contains dynamically linked ELF binaries, but RPM" \ + "requires metadata is missing libc soname dependency (libc.so.6*)" fi } @@ -298,6 +312,10 @@ for rpm_file in "${rpms[@]}"; do done if [ "${errors}" -ne 0 ]; then + if [ "${MODE}" = "WARNING" ]; then + echo "RPM validation completed with ${errors} warning(s)" + exit 0 + fi echo "RPM validation failed with ${errors} error(s)" exit 1 fi diff --git a/utils/docker/Dockerfile.el.9 b/utils/docker/Dockerfile.el.9 index 44e89745fdc..1079d88263b 100644 --- a/utils/docker/Dockerfile.el.9 +++ b/utils/docker/Dockerfile.el.9 @@ -8,13 +8,22 @@ # # Use --target build-ci to create image with basic build environment for gcc +# NOTE +# Java build is disabled by default on EL9 +# Do not change this line !!! +ARG DAOS_JAVA_BUILD=no + # Global defaults -ARG DAOS_TARGET_TYPE=release ARG DAOS_DEPS_BUILD=yes ARG DAOS_KEEP_BUILD=no ARG DAOS_PACKAGES_BUILD=yes -ARG DEPS_JOBS=32 -ARG DAOS_PACKAGES_VERIFY=no +ARG INSTALL_SCRIPT_SUFFIX=el9.sh +# Left empty on purpose: scons and the build scripts provide their own defaults. +ARG DAOS_PACKAGES_VERIFY= +ARG DAOS_TARGET_TYPE= +ARG DEPS_JOBS= + +ARG DAOS_BUILD_TYPE=$DAOS_TARGET_TYPE # Pull base image ARG POINT_RELEASE= @@ -23,7 +32,6 @@ FROM $BASE_DISTRO AS basic LABEL maintainer="daos@daos.groups.io" # Needed for later use of BASE_DISTRO ARG BASE_DISTRO - # Intermittent cache-bust. Refresh base image at least once a week. ARG CB0 @@ -59,7 +67,8 @@ RUN set -e; \ fi # Setup local repo if available -RUN --mount=type=bind,src=./utils/scripts/helpers/repo-helper-el9.sh,target=/tmp/daos/repo-helper.sh \ +ARG INSTALL_SCRIPT_SUFFIX +RUN --mount=type=bind,src=./utils/scripts/helpers/repo-helper-$INSTALL_SCRIPT_SUFFIX,target=/tmp/daos/repo-helper.sh \ /tmp/daos/repo-helper.sh # Remove own DAOS dependencies from the system to avoid conflicts with @@ -79,7 +88,8 @@ FROM basic AS build-ci # Install minimal set of packages for building DAOS with gcc compiler and # no Java support. -RUN --mount=type=bind,source=utils/scripts/install-el9.sh,target=/tmp/daos/install.sh \ +ARG INSTALL_SCRIPT_SUFFIX +RUN --mount=type=bind,source=utils/scripts/install-$INSTALL_SCRIPT_SUFFIX,target=/tmp/daos/install.sh \ INSTALL_BUILD_CI_ONLY="true" /tmp/daos/install.sh && \ dnf clean all @@ -108,13 +118,15 @@ RUN --mount=type=bind,source=requirements-build.txt,target=/tmp/daos/requirement # Install DAOS dependencies if available ARG DAOS_DEPS_INSTALL=yes -RUN --mount=type=bind,source=ci/rpm/install_deps.sh,target=/tmp/daos/ci/rpm/install_deps.sh \ - --mount=type=bind,source=ci/parse_ci_envs.sh,target=/tmp/daos/ci/parse_ci_envs.sh \ +ARG DAOS_DEPS_EXT_REPO= +RUN --mount=type=bind,source=utils/build/install_deps.sh,target=/tmp/daos/utils/build/install_deps.sh \ + --mount=type=bind,source=utils/build/build_utils.sh,target=/tmp/daos/utils/build/build_utils.sh \ --mount=type=bind,source=utils/rpms/package_version.sh,target=/tmp/daos/utils/rpms/package_version.sh \ --mount=type=bind,source=utils/rpms/package_info.sh,target=/tmp/daos/utils/rpms/package_info.sh \ --mount=type=bind,source=utils/rpms/daos.spec,target=/tmp/daos/utils/rpms/daos.spec \ [ "$DAOS_DEPS_INSTALL" != "yes" ] || { \ - /tmp/daos/ci/rpm/install_deps.sh el9; \ + export DAOS_DEPS_EXT_REPO="${DAOS_DEPS_EXT_REPO}" && \ + /tmp/daos/utils/build/install_deps.sh "el9"; \ sudo dnf clean all; \ } @@ -124,7 +136,7 @@ ARG DAOS_DEPS_BUILD ARG DEPS_JOBS ARG DAOS_TARGET_TYPE ARG DAOS_KEEP_BUILD - +ARG DAOS_BUILD_TYPE RUN --mount=type=bind,source=SConstruct,target=/home/daos/pre/SConstruct \ --mount=type=bind,source=deps,target=/home/daos/pre/deps \ --mount=type=bind,source=site_scons/prereq_tools,target=/home/daos/pre/site_scons/prereq_tools \ @@ -132,36 +144,35 @@ RUN --mount=type=bind,source=SConstruct,target=/home/daos/pre/SConstruct \ --mount=type=bind,source=utils/build.config,target=/home/daos/pre/utils/build.config \ --mount=type=bind,source=utils/scripts/copy_files.sh,target=/home/daos/pre/utils/scripts/copy_files.sh \ --mount=type=bind,source=utils/scripts/create_spdk_pkgconfig.sh,target=/home/daos/pre/utils/scripts/create_spdk_pkgconfig.sh \ - --mount=type=bind,source=ci/rpm/build_deps.sh,target=/home/daos/pre/utils/scripts/build_deps.sh \ + --mount=type=bind,source=utils/build/build_deps.sh,target=/home/daos/pre/utils/build/build_deps.sh \ + --mount=type=bind,source=utils/build/build_utils.sh,target=/home/daos/pre/utils/build/build_utils.sh \ + --mount=type=bind,source=utils/rpms,target=/home/daos/pre/utils/rpms \ [ "$DAOS_DEPS_BUILD" != "yes" ] || { \ - DAOS_DEPS_JOBS="$DEPS_JOBS" DAOS_TARGET_TYPE="$DAOS_TARGET_TYPE" \ - utils/scripts/build_deps.sh && \ + utils/build/build_deps.sh \ + ${DEPS_JOBS:+--jobs "$DEPS_JOBS"} \ + ${DAOS_BUILD_TYPE:+BUILD_TYPE="$DAOS_BUILD_TYPE"} \ + ${DAOS_TARGET_TYPE:+TARGET_TYPE="$DAOS_TARGET_TYPE"} && \ + [ "$DAOS_KEEP_BUILD" != "no" ] || { \ rm -rf .sconf_temp .sconsign.dblite config.log && \ - ([ "$DAOS_KEEP_BUILD" != "no" ] || /bin/rm -rf build *.gz); \ + /bin/rm -rf build *.gz; \ + } ; \ } # Build DAOS dependencies RPMs ARG DAOS_PACKAGES_BUILD ARG DAOS_PACKAGES_VERIFY RUN --mount=type=bind,source=utils/rpms,target=/home/daos/pre/utils/rpms \ + --mount=type=bind,source=utils/build,target=/home/daos/pre/utils/build \ --mount=type=bind,source=utils/sl/setup_local.sh,target=/home/daos/pre/utils/sl/setup_local.sh \ --mount=type=bind,source=utils/build.config,target=/home/daos/pre/utils/build.config \ [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_DEPS_BUILD" != "yes" ] || \ { \ export DISTRO="el9" && \ - /home/daos/pre/utils/rpms/build_packages.sh deps "$DAOS_PACKAGES_VERIFY" && \ - mkdir -p /home/daos/rpms/deps && \ - find . -maxdepth 1 -type f -name '*.rpm' \ - -exec mv -t /home/daos/rpms/deps {} + ; \ + export RPM_OUTPUT_DIR=/home/daos/rpms && \ + utils/build/build_packages.sh deps "$DAOS_PACKAGES_VERIFY" ; \ } -RUN --mount=type=bind,source=utils/rpms,target=/home/daos/pre/utils/rpms \ - [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_DEPS_BUILD" != "yes" ] || \ - { \ - [ "$DAOS_PACKAGES_VERIFY" = "yes" ] || { \ - utils/rpms/verify_rpms.sh /home/daos/rpms/deps "el9" || true; \ - } \ - } +WORKDIR /home/daos FROM build-ci AS build-local # Set a label. This is useful for searching for DAOS images, but is also used @@ -171,13 +182,13 @@ LABEL DAOS=true ARG COMPILER=gcc ARG DAOS_DEPS_BUILD ARG DAOS_BUILD=$DAOS_DEPS_BUILD -ARG DAOS_JAVA_BUILD=$DAOS_BUILD +ARG DAOS_JAVA_BUILD=no # Install the rest of components if other than gcc compiler will be used for # DAOS or Java build is requested. # This is done in a separate stage to allow for a smaller image for CI building only. USER root:root -RUN --mount=type=bind,source=utils/scripts/install-el9.sh,target=/tmp/daos/install.sh \ +RUN --mount=type=bind,source=utils/scripts/install-$INSTALL_SCRIPT_SUFFIX,target=/tmp/daos/install.sh \ ([ "$COMPILER" = "gcc" ] && [ "$DAOS_JAVA_BUILD" != "yes" ]) || { \ /tmp/daos/install.sh && \ dnf clean all; \ @@ -194,7 +205,7 @@ ARG DEPS_JOBS ARG JOBS=$DEPS_JOBS ARG DAOS_TARGET_TYPE ARG DAOS_KEEP_BUILD -ARG DAOS_BUILD_TYPE=$DAOS_TARGET_TYPE +ARG DAOS_BUILD_TYPE ARG DAOS_KEEP_SRC=no # We need to copy the source code into the image as we need read/write copy @@ -221,20 +232,24 @@ RUN --mount=type=bind,source=VERSION,target=/home/daos/daos/VERSION \ --mount=type=bind,source=utils/certs,target=/home/daos/daos/utils/certs \ --mount=type=bind,source=utils/completion,target=/home/daos/daos/utils/completion \ --mount=type=bind,source=utils/config,target=/home/daos/daos/utils/config \ + --mount=type=bind,source=utils/build/build_daos.sh,target=/home/daos/daos/utils/build/build_daos.sh \ + --mount=type=bind,source=utils/build/build_utils.sh,target=/home/daos/daos/utils/build/build_utils.sh \ + --mount=type=bind,source=utils/rpms,target=/home/daos/daos/rpms \ --mount=type=bind,source=src,target=/tmp/daos-src \ [ "$DAOS_BUILD" != "yes" ] || { \ export GOCACHE=/tmp/go-cache && \ mkdir -p /home/daos/daos/src && \ cp -r /tmp/daos-src/. /home/daos/daos/src && \ - scons --jobs $JOBS install PREFIX=/opt/daos COMPILER=$COMPILER \ - BUILD_TYPE=$DAOS_BUILD_TYPE TARGET_TYPE=$DAOS_TARGET_TYPE \ - USE_INSTALLED=all && \ + utils/build/build_daos.sh \ + ${JOBS:+--jobs "$JOBS"} \ + ${COMPILER:+COMPILER=$COMPILER} \ + ${DAOS_BUILD_TYPE:+BUILD_TYPE="$DAOS_BUILD_TYPE"} \ + ${DAOS_TARGET_TYPE:+TARGET_TYPE="$DAOS_TARGET_TYPE"} && \ ([ "$DAOS_KEEP_BUILD" != "no" ] || { \ - /bin/rm -rf build; \ + rm -rf .sconf_temp .sconsign.dblite config.log build; \ }) && \ go clean -cache && \ cp -r utils/config/examples /opt/daos && \ - rm -rf .sconf_temp .sconsign.dblite config.log && \ [ "$DAOS_KEEP_SRC" != "no" ] || { \ rm -rf /home/daos/daos/src; \ }; \ @@ -254,45 +269,38 @@ USER daos_server:daos_server ARG DAOS_PACKAGES_BUILD ARG DAOS_PACKAGES_VERIFY RUN --mount=type=bind,source=utils/rpms,target=/home/daos/daos/utils/rpms \ + --mount=type=bind,source=utils/build,target=/home/daos/daos/utils/build \ --mount=type=bind,source=utils/sl,target=/home/daos/daos/utils/sl \ --mount=type=bind,source=utils/systemd,target=/home/daos/daos/utils/systemd \ --mount=type=bind,source=utils/build.config,target=/home/daos/daos/utils/build.config \ - --mount=type=bind,source=utils/rpms/build_packages.sh,target=/home/daos/daos/utils/rpms/build_packages.sh \ [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_BUILD" != "yes" ] || { \ export DISTRO="el9" && \ - /home/daos/daos/utils/rpms/build_packages.sh daos "$DAOS_PACKAGES_VERIFY" && \ - mkdir -p /home/daos/rpms/daos && \ - find . -maxdepth 1 -type f -name '*.rpm' \ - -exec mv -t /home/daos/rpms/daos {} + ; \ + export RPM_OUTPUT_DIR=/home/daos/rpms && \ + /home/daos/daos/utils/build/build_packages.sh daos "$DAOS_PACKAGES_VERIFY" ; \ } -RUN --mount=type=bind,source=utils/rpms,target=/home/daos/daos/utils/rpms \ - [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_BUILD" != "yes" ] || { \ - [ "$DAOS_PACKAGES_VERIFY" = "yes" ] || { \ - utils/rpms/verify_rpms.sh /home/daos/rpms/daos "el9" || true; \ - } \ +# Build java and hadoop bindings +USER daos_server:daos_server +RUN [ "$DAOS_JAVA_BUILD" != "yes" ] || { \ + mkdir -p /home/daos/.m2; \ +} + +RUN --mount=type=bind,source=utils/scripts/helpers/maven-settings.xml.in,target=/home/daos/.m2/settings.xml \ + --mount=type=bind,source=src,target=/tmp/daos-src \ + [ "$DAOS_BUILD" != "yes" ] || [ "$DAOS_JAVA_BUILD" != "yes" ] || { \ + mkdir -p /home/daos/daos/src && \ + cp -r /tmp/daos-src/. /home/daos/daos/src && \ + pushd /home/daos/daos/src/client/java && \ + mvn clean install -ntp -T 1C -DskipITs -Dgpg.skip \ + -Ddaos.install.path=/opt/daos && \ + popd && \ + rm -rf /home/daos/.m2/* && \ + [ "$DAOS_KEEP_SRC" != "no" ] || { \ + rm -rf /home/daos/daos/src; \ + }; \ } -# Build java and hadoop bindings -#USER daos_server:daos_server -#RUN [ "$DAOS_JAVA_BUILD" != "yes" ] || { \ -# mkdir -p /home/daos/.m2; \ -#} - -#RUN --mount=type=bind,source=utils/scripts/helpers/maven-settings.xml.in,target=/home/daos/.m2/settings.xml \ -# --mount=type=bind,source=src,target=/tmp/daos-src \ -# [ "$DAOS_BUILD" != "yes" ] || [ "$DAOS_JAVA_BUILD" != "yes" ] || { \ -# mkdir -p /home/daos/daos/src && \ -# cp -r /tmp/daos-src/. /home/daos/daos/src && \ -# pushd /home/daos/daos/src/client/java && \ -# mvn clean install -ntp -T 1C -DskipITs -Dgpg.skip \ -# -Ddaos.install.path=/opt/daos && \ -# popd && \ -# rm -rf /home/daos/.m2/* && \ -# [ "$DAOS_KEEP_SRC" != "no" ] || { \ -# rm -rf /home/daos/daos/src; \ -# }; \ -# } +WORKDIR /home/daos # Setup docker image for Unit Tests FROM build-local AS full @@ -300,7 +308,8 @@ FROM build-local AS full # This is done in a separate stage to allow for a smaller image for CI # and local build USER root:root -RUN --mount=type=bind,source=utils/scripts/install-el9.sh,target=/tmp/daos/install.sh \ +ARG INSTALL_SCRIPT_SUFFIX +RUN --mount=type=bind,source=utils/scripts/install-$INSTALL_SCRIPT_SUFFIX,target=/tmp/daos/install.sh \ /tmp/daos/install.sh && \ dnf clean all diff --git a/utils/docker/Dockerfile.leap.15 b/utils/docker/Dockerfile.leap.15 index 621faa8f2d0..0f228b1a44a 100644 --- a/utils/docker/Dockerfile.leap.15 +++ b/utils/docker/Dockerfile.leap.15 @@ -3,18 +3,22 @@ # Copyright 2025-2026 Hewlett Packard Enterprise Development LP # All rights reserved. # -# 'recipe' for Docker to build an image of Leap based +# 'recipe' for Docker to build an image of Leap 15 based # environment for building the DAOS project. # # Use --target build-ci to create image with basic build environment for gcc # Global defaults -ARG DAOS_TARGET_TYPE=release ARG DAOS_DEPS_BUILD=yes ARG DAOS_KEEP_BUILD=no ARG DAOS_PACKAGES_BUILD=yes -ARG DEPS_JOBS=32 -ARG DAOS_PACKAGES_VERIFY=no +ARG INSTALL_SCRIPT_SUFFIX=leap15.sh +# Left empty on purpose: scons and the build scripts provide their own defaults. +ARG DAOS_PACKAGES_VERIFY= +ARG DAOS_TARGET_TYPE= +ARG DEPS_JOBS= + +ARG DAOS_BUILD_TYPE=$DAOS_TARGET_TYPE # Pull base image ARG POINT_RELEASE= @@ -23,7 +27,6 @@ FROM $BASE_DISTRO AS basic LABEL maintainer="daos@daos.groups.io" # Needed for later use of BASE_DISTRO ARG BASE_DISTRO - # Intermittent cache-bust. Refresh base image at least once a week. ARG CB0 @@ -59,7 +62,8 @@ RUN set -e; \ fi # Setup local repo if available -RUN --mount=type=bind,src=./utils/scripts/helpers/repo-helper-leap15.sh,target=/tmp/daos/repo-helper.sh \ +ARG INSTALL_SCRIPT_SUFFIX +RUN --mount=type=bind,src=./utils/scripts/helpers/repo-helper-$INSTALL_SCRIPT_SUFFIX,target=/tmp/daos/repo-helper.sh \ /tmp/daos/repo-helper.sh # Remove own DAOS dependencies from the system to avoid conflicts with @@ -79,9 +83,13 @@ FROM basic AS build-ci # Install minimal set of packages for building DAOS with gcc compiler and # no Java support. -RUN --mount=type=bind,source=utils/scripts/install-leap15.sh,target=/tmp/daos/install.sh \ +ARG INSTALL_SCRIPT_SUFFIX +RUN --mount=type=bind,source=utils/scripts/install-$INSTALL_SCRIPT_SUFFIX,target=/tmp/daos/install.sh \ INSTALL_BUILD_CI_ONLY="true" /tmp/daos/install.sh && \ dnf clean all +# Local container builds need the OpenMPI wrappers on PATH; Jenkins loads MPI +# through Lmod before invoking scons. +# ENV PATH=/usr/lib64/mpi/gcc/openmpi3/bin:$PATH # According to https://pkgs.org/search/?q=lua-lmod, Leap 15.6 only has lua-lmod-8.7.34. # This version has a problem with loading modules, as described in https://github.com/TACC/Lmod/issues/687. @@ -124,14 +132,16 @@ RUN --mount=type=bind,source=requirements-build.txt,target=/tmp/daos/requirement # Install DAOS dependencies if available ARG DAOS_DEPS_INSTALL=yes +ARG DAOS_DEPS_EXT_REPO= ARG POINT_RELEASE -RUN --mount=type=bind,source=ci/rpm/install_deps.sh,target=/tmp/daos/ci/rpm/install_deps.sh \ - --mount=type=bind,source=ci/parse_ci_envs.sh,target=/tmp/daos/ci/parse_ci_envs.sh \ +RUN --mount=type=bind,source=utils/build/install_deps.sh,target=/tmp/daos/utils/build/install_deps.sh \ + --mount=type=bind,source=utils/build/build_utils.sh,target=/tmp/daos/utils/build/build_utils.sh \ --mount=type=bind,source=utils/rpms/package_version.sh,target=/tmp/daos/utils/rpms/package_version.sh \ --mount=type=bind,source=utils/rpms/package_info.sh,target=/tmp/daos/utils/rpms/package_info.sh \ --mount=type=bind,source=utils/rpms/daos.spec,target=/tmp/daos/utils/rpms/daos.spec \ [ "$DAOS_DEPS_INSTALL" != "yes" ] || { \ - /tmp/daos/ci/rpm/install_deps.sh suse.lp15${POINT_RELEASE#\.}; \ + export DAOS_DEPS_EXT_REPO="${DAOS_DEPS_EXT_REPO}" && \ + /tmp/daos/utils/build/install_deps.sh "suse.lp15${POINT_RELEASE#\.}"; \ sudo dnf clean all; \ } @@ -141,7 +151,7 @@ ARG DAOS_DEPS_BUILD ARG DEPS_JOBS ARG DAOS_TARGET_TYPE ARG DAOS_KEEP_BUILD - +ARG DAOS_BUILD_TYPE RUN --mount=type=bind,source=SConstruct,target=/home/daos/pre/SConstruct \ --mount=type=bind,source=deps,target=/home/daos/pre/deps \ --mount=type=bind,source=site_scons/prereq_tools,target=/home/daos/pre/site_scons/prereq_tools \ @@ -149,37 +159,35 @@ RUN --mount=type=bind,source=SConstruct,target=/home/daos/pre/SConstruct \ --mount=type=bind,source=utils/build.config,target=/home/daos/pre/utils/build.config \ --mount=type=bind,source=utils/scripts/copy_files.sh,target=/home/daos/pre/utils/scripts/copy_files.sh \ --mount=type=bind,source=utils/scripts/create_spdk_pkgconfig.sh,target=/home/daos/pre/utils/scripts/create_spdk_pkgconfig.sh \ - --mount=type=bind,source=ci/rpm/build_deps.sh,target=/home/daos/pre/utils/scripts/build_deps.sh \ + --mount=type=bind,source=utils/build/build_deps.sh,target=/home/daos/pre/utils/build/build_deps.sh \ + --mount=type=bind,source=utils/build/build_utils.sh,target=/home/daos/pre/utils/build/build_utils.sh \ + --mount=type=bind,source=utils/rpms,target=/home/daos/pre/utils/rpms \ [ "$DAOS_DEPS_BUILD" != "yes" ] || { \ - DAOS_DEPS_JOBS="$DEPS_JOBS" DAOS_TARGET_TYPE="$DAOS_TARGET_TYPE" \ - utils/scripts/build_deps.sh && \ - rm -rf .sconf_temp .sconsign.dblite config.log && \ - ([ "$DAOS_KEEP_BUILD" != "no" ] || /bin/rm -rf build *.gz); \ + utils/build/build_deps.sh \ + ${DEPS_JOBS:+--jobs "$DEPS_JOBS"} \ + ${DAOS_BUILD_TYPE:+BUILD_TYPE="$DAOS_BUILD_TYPE"} \ + ${DAOS_TARGET_TYPE:+TARGET_TYPE="$DAOS_TARGET_TYPE"} && \ + [ "$DAOS_KEEP_BUILD" != "no" ] || { \ + rm -rf .sconf_temp .sconsign.dblite config.log && \ + /bin/rm -rf build *.gz; \ + } ; \ } # Build DAOS dependencies RPMs ARG DAOS_PACKAGES_BUILD -ARG POINT_RELEASE ARG DAOS_PACKAGES_VERIFY RUN --mount=type=bind,source=utils/rpms,target=/home/daos/pre/utils/rpms \ + --mount=type=bind,source=utils/build,target=/home/daos/pre/utils/build \ --mount=type=bind,source=utils/sl/setup_local.sh,target=/home/daos/pre/utils/sl/setup_local.sh \ --mount=type=bind,source=utils/build.config,target=/home/daos/pre/utils/build.config \ [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_DEPS_BUILD" != "yes" ] || \ { \ export DISTRO="suse.lp15${POINT_RELEASE#\.}" && \ - /home/daos/pre/utils/rpms/build_packages.sh deps "$DAOS_PACKAGES_VERIFY" && \ - mkdir -p /home/daos/rpms/deps && \ - find . -maxdepth 1 -type f -name '*.rpm' \ - -exec mv -t /home/daos/rpms/deps {} + ; \ + export RPM_OUTPUT_DIR=/home/daos/rpms && \ + utils/build/build_packages.sh deps "$DAOS_PACKAGES_VERIFY" ; \ } -RUN --mount=type=bind,source=utils/rpms,target=/home/daos/pre/utils/rpms \ - [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_DEPS_BUILD" != "yes" ] || \ - { \ - [ "$DAOS_PACKAGES_VERIFY" = "yes" ] || { \ - utils/rpms/verify_rpms.sh /home/daos/rpms/deps "suse.lp15${POINT_RELEASE#\.}" || true; \ - } \ - } +WORKDIR /home/daos FROM build-ci AS build-local # Set a label. This is useful for searching for DAOS images, but is also used @@ -189,13 +197,13 @@ LABEL DAOS=true ARG COMPILER=gcc ARG DAOS_DEPS_BUILD ARG DAOS_BUILD=$DAOS_DEPS_BUILD -ARG DAOS_JAVA_BUILD=$DAOS_BUILD +ARG DAOS_JAVA_BUILD=no # Install the rest of components if other than gcc compiler will be used for # DAOS or Java build is requested. # This is done in a separate stage to allow for a smaller image for CI building only. USER root:root -RUN --mount=type=bind,source=utils/scripts/install-leap15.sh,target=/tmp/daos/install.sh \ +RUN --mount=type=bind,source=utils/scripts/install-$INSTALL_SCRIPT_SUFFIX,target=/tmp/daos/install.sh \ ([ "$COMPILER" = "gcc" ] && [ "$DAOS_JAVA_BUILD" != "yes" ]) || { \ /tmp/daos/install.sh && \ dnf clean all; \ @@ -212,7 +220,7 @@ ARG DEPS_JOBS ARG JOBS=$DEPS_JOBS ARG DAOS_TARGET_TYPE ARG DAOS_KEEP_BUILD -ARG DAOS_BUILD_TYPE=$DAOS_TARGET_TYPE +ARG DAOS_BUILD_TYPE ARG DAOS_KEEP_SRC=no # We need to copy the source code into the image as we need read/write copy @@ -239,20 +247,24 @@ RUN --mount=type=bind,source=VERSION,target=/home/daos/daos/VERSION \ --mount=type=bind,source=utils/certs,target=/home/daos/daos/utils/certs \ --mount=type=bind,source=utils/completion,target=/home/daos/daos/utils/completion \ --mount=type=bind,source=utils/config,target=/home/daos/daos/utils/config \ + --mount=type=bind,source=utils/build/build_daos.sh,target=/home/daos/daos/utils/build/build_daos.sh \ + --mount=type=bind,source=utils/build/build_utils.sh,target=/home/daos/daos/utils/build/build_utils.sh \ + --mount=type=bind,source=utils/rpms,target=/home/daos/daos/rpms \ --mount=type=bind,source=src,target=/tmp/daos-src \ [ "$DAOS_BUILD" != "yes" ] || { \ export GOCACHE=/tmp/go-cache && \ mkdir -p /home/daos/daos/src && \ cp -r /tmp/daos-src/. /home/daos/daos/src && \ - scons --jobs $JOBS install PREFIX=/opt/daos COMPILER=$COMPILER \ - BUILD_TYPE=$DAOS_BUILD_TYPE TARGET_TYPE=$DAOS_TARGET_TYPE \ - USE_INSTALLED=all && \ + utils/build/build_daos.sh \ + ${JOBS:+--jobs "$JOBS"} \ + ${COMPILER:+COMPILER=$COMPILER} \ + ${DAOS_BUILD_TYPE:+BUILD_TYPE="$DAOS_BUILD_TYPE"} \ + ${DAOS_TARGET_TYPE:+TARGET_TYPE="$DAOS_TARGET_TYPE"} && \ ([ "$DAOS_KEEP_BUILD" != "no" ] || { \ - /bin/rm -rf build; \ + rm -rf .sconf_temp .sconsign.dblite config.log build; \ }) && \ go clean -cache && \ cp -r utils/config/examples /opt/daos && \ - rm -rf .sconf_temp .sconsign.dblite config.log && \ [ "$DAOS_KEEP_SRC" != "no" ] || { \ rm -rf /home/daos/daos/src; \ }; \ @@ -270,26 +282,17 @@ USER daos_server:daos_server # Build DAOS RPMs ARG DAOS_PACKAGES_BUILD -ARG POINT_RELEASE ARG DAOS_PACKAGES_VERIFY +ARG POINT_RELEASE RUN --mount=type=bind,source=utils/rpms,target=/home/daos/daos/utils/rpms \ + --mount=type=bind,source=utils/build,target=/home/daos/daos/utils/build \ --mount=type=bind,source=utils/sl,target=/home/daos/daos/utils/sl \ --mount=type=bind,source=utils/systemd,target=/home/daos/daos/utils/systemd \ --mount=type=bind,source=utils/build.config,target=/home/daos/daos/utils/build.config \ - --mount=type=bind,source=utils/rpms/build_packages.sh,target=/home/daos/daos/utils/rpms/build_packages.sh \ [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_BUILD" != "yes" ] || { \ export DISTRO="suse.lp15${POINT_RELEASE#\.}" && \ - /home/daos/daos/utils/rpms/build_packages.sh daos "$DAOS_PACKAGES_VERIFY" && \ - mkdir -p /home/daos/rpms/daos && \ - find . -maxdepth 1 -type f -name '*.rpm' \ - -exec mv -t /home/daos/rpms/daos {} + ; \ - } - -RUN --mount=type=bind,source=utils/rpms,target=/home/daos/daos/utils/rpms \ - [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_BUILD" != "yes" ] || { \ - [ "$DAOS_PACKAGES_VERIFY" = "yes" ] || { \ - utils/rpms/verify_rpms.sh /home/daos/rpms/daos "suse.lp15${POINT_RELEASE#\.}" || true; \ - } \ + export RPM_OUTPUT_DIR=/home/daos/rpms && \ + /home/daos/daos/utils/build/build_packages.sh daos "$DAOS_PACKAGES_VERIFY" ; \ } # Build java and hadoop bindings @@ -313,13 +316,16 @@ RUN --mount=type=bind,source=utils/scripts/helpers/maven-settings.xml.in,target= }; \ } + WORKDIR /home/daos + # Setup docker image for Unit Tests FROM build-local AS full # Install the rest of components if not already installed. # This is done in a separate stage to allow for a smaller image for CI # and local build USER root:root -RUN --mount=type=bind,source=utils/scripts/install-leap15.sh,target=/tmp/daos/install.sh \ +ARG INSTALL_SCRIPT_SUFFIX +RUN --mount=type=bind,source=utils/scripts/install-$INSTALL_SCRIPT_SUFFIX,target=/tmp/daos/install.sh \ /tmp/daos/install.sh && \ dnf clean all diff --git a/utils/docker/Dockerfile.ubuntu b/utils/docker/Dockerfile.ubuntu index 3a881f40783..d1049ba4385 100644 --- a/utils/docker/Dockerfile.ubuntu +++ b/utils/docker/Dockerfile.ubuntu @@ -126,6 +126,7 @@ RUN [ "$DAOS_DEPS_BUILD" != "yes" ] || \ ([ "$DAOS_KEEP_BUILD" != "no" ] || /bin/rm -rf build *.gz); } COPY --chown=daos_server:daos_server utils/rpms utils/rpms +COPY --chown=daos_server:daos_server utils/build utils/build COPY --chown=daos_server:daos_server utils/sl utils/sl # Build third party RPMs @@ -133,7 +134,7 @@ RUN [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_DEPS_BUILD" != "yes" ] || { export DISTRO=ubuntu && \ export OUTPUT_TYPE=deb && \ export LIBDIR=/usr/lib/x86_64-linux-gnu && \ - utils/rpms/build_packages.sh deps && \ + utils/build/build_packages.sh deps && \ mkdir -p /home/daos/dpkg && \ cp *.deb /home/daos/dpkg; \ } @@ -180,7 +181,7 @@ RUN [ "$DAOS_PACKAGES_BUILD" != "yes" ] || [ "$DAOS_BUILD" != "yes" ] || { \ export DISTRO=ubuntu && \ export OUTPUT_TYPE=deb && \ export LIBDIR=/usr/lib/x86_64-linux-gnu && \ - utils/rpms/build_packages.sh daos && \ + utils/build/build_packages.sh daos && \ mkdir -p /home/daos/dpkg && \ cp *.deb /home/daos/dpkg; \ } diff --git a/utils/rpms/build_packages.sh b/utils/rpms/build_packages.sh deleted file mode 100755 index 53812b02d40..00000000000 --- a/utils/rpms/build_packages.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -# Copyright 2025 Google LLC -# Copyright 2026 Hewlett Packard Enterprise Development LP -# -# SPDX-License-Identifier: BSD-2-Clause-Patent - -# Usage: -# build_packages.sh [build_type] [verify] -# -# Args: -# build_type What to build: deps|daos|all. Default: all -# verify Run verify_rpms.sh after package build: yes|no. Default: no -set -eEuo pipefail - -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -repo_root="$(cd "${script_dir}/../.." >/dev/null 2>&1 && pwd)" - -build_type="${1:-all}" -verify_rpms="${2:-no}" - -case "${verify_rpms}" in - yes|no) - ;; - *) - echo "ERROR: verify parameter must be 'yes' or 'no' (got: ${verify_rpms})" - exit 1 - ;; -esac - -cd "${repo_root}" -source utils/sl/setup_local.sh -if [[ "${build_type}" =~ deps|all ]]; then - utils/rpms/argobots.sh - utils/rpms/fused.sh - utils/rpms/isa-l.sh - utils/rpms/isa-l_crypto.sh - utils/rpms/libfabric.sh - utils/rpms/mercury.sh - utils/rpms/pmdk.sh - utils/rpms/daos-spdk.sh -fi -if [[ "${build_type}" =~ daos|all ]]; then - utils/rpms/daos.sh -fi - -case "${DISTRO:-el9}" in - el*|suse.lp15*) - if [ "${OUTPUT_TYPE:-rpm}" = "rpm" ] && [ "${verify_rpms}" = "yes" ]; then - "${script_dir}/verify_rpms.sh" "$PWD" "${DISTRO:-el9}" - fi - ;; -esac diff --git a/utils/rpms/fpm_common.sh b/utils/rpms/fpm_common.sh index 9476f21cf4c..c6704fbc32c 100644 --- a/utils/rpms/fpm_common.sh +++ b/utils/rpms/fpm_common.sh @@ -174,6 +174,10 @@ build_package() { config_files=() create_opts "--config-files" config_files "${CONFIG_FILES[@]}" pkgname="${name}-${VERSION}-${RELEASE}.${ARCH}.${output_type}" + if [ -n "${PACKAGE_OUTPUT_DIR:-}" ]; then + mkdir -p "${PACKAGE_OUTPUT_DIR}" + pkgname="${PACKAGE_OUTPUT_DIR}/${pkgname}" + fi rm -f "${pkgname}" # shellcheck disable=SC2068 fpm --verbose -s "${PACKAGE_TYPE}" -t "${output_type}" \