diff --git a/Jenkinsfile b/Jenkinsfile index b0a58c7..2c0cf23 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,25 +1,50 @@ #!/usr/bin/env groovy -// Copyright (c) 2018 Intel Corporation -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +/* Copyright (c) 2018-2019 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +// To use a test branch (i.e. PR) until it lands to master +// I.e. for testing library changes +@Library(value="pipeline-lib@scons-to-logfile") _ -@Library(value="pipeline-lib") _ +def sanitized_JOB_NAME = JOB_NAME.toLowerCase().replaceAll('/', '-').replaceAll('%2f', '-') + +String get_deps_build_vars() { + def deps = [ + 'MERCURY' : '1.0.1-2', + 'OMPI' : '3.0.0rc4-3', + 'PMIX' : '2.1.1-2', + 'LIBFABRIC': '1.7.1rc1-1', + 'OPENPA' : '1.0.4-2' + ] + def buildargs = "" + deps.each { + dep, ver -> buildargs +="--build-arg ${dep}=" + + dep.toLowerCase() + "-${ver} " + } + + return buildargs +} + +def log_date_re = '[0-9]{4}-[0-9]{2}-[0-9]{1,2}' +def log_time_re = '[0-9]{2}:[0-9]{2}:[0-9]{2}\\.[0-9]{1,3}Z' +def log_timestamp_re = '\\[' + log_date_re + 'T' + log_time_re + '\\]' pipeline { agent any @@ -91,209 +116,248 @@ pipeline { } stages { - stage('Phase 1') { - parallel { - stage ('environment report') { - agent any - steps { - sh "export; pip freeze || true" - } + stage('Cancel Previous Builds') { + when { changeRequest() } + steps { + cancelPreviousBuilds() } - stage ('code_lint_check') { - - // Just use CentOS for lint checks. - agent { - dockerfile { - filename "${params.PDOCKERFILE ? params.PDOCKERFILE : 'Dockerfile.centos.7'}" - dir 'scons_local/docker' - label 'docker_runner' - additionalBuildArgs '$BUILDARGS' - } - } // agent - steps { - sh "pip freeze || true" - checkPatch review_creds: "${env.GITHUB_CREDS}" - } // steps - } // stage ('code_lint_check') - stage ('fuse build') { - agent { - dockerfile { - filename "${params.PDOCKERFILE ? params.PDOCKERFILE : 'Dockerfile.centos.7'}" - dir 'scons_local/docker' - label 'docker_runner' - additionalBuildArgs '$BUILDARGS' - } - } // agent - steps { - // Older scons_local_review only used master branch. - // Newer one looked up the last known good build of master branch. - // Pipeline currently does not have access that info. - - sh 'rm -rf testbin/ && mkdir -p testbin' - sconsBuild target: 'fuse', - directory: 'scons_local', - scm: [url: 'https://github.com/libfuse/libfuse.git', - branch: "${env.FUSE_COMMIT}", - cleanAfterCheckout: true], - no_install: true, // No separate install step - TARGET_PREFIX: '/testbin', - target_work: 'testbin' - - echo "fuse build succeeded" - } // steps - } // stage ('fuse build') - stage ('openpa prebuild') { - agent { - dockerfile { - filename "${params.PDOCKERFILE ? params.PDOCKERFILE : 'Dockerfile.centos.7'}" - dir 'scons_local/docker' - label 'docker_runner' - additionalBuildArgs '$BUILDARGS' - } - } // agent - steps { - sh 'rm -rf testbin/ && mkdir -p testbin' - sconsBuild target: 'openpa', - directory: 'scons_local', - scm: [url: 'http://git.mcs.anl.gov/radix/openpa.git', - branch: "${env.OPENPA_COMMIT}", - cleanAfterCheckout: true], - no_install: true, // No separate install step - TARGET_PREFIX: '/testbin', - target_work: 'testbin' - - echo "openpa build succeeded" - stash name: "${env.MY_DOCKERFILE}-openpa", - includes: 'testbin/openpa/**' - } // steps - } // stage ('openpa prebuild') - stage ('hwloc prebuild') { - agent { - dockerfile { - filename "${params.PDOCKERFILE ? params.PDOCKERFILE : 'Dockerfile.centos.7'}" - dir 'scons_local/docker' - label 'docker_runner' - additionalBuildArgs '$BUILDARGS' + } + stage('Phase 1') { + parallel { + stage ('environment report') { + agent any + steps { + sh "export; pip freeze || true" } - } // agent - steps { - sh 'rm -rf testbin/ && mkdir -p testbin' - sconsBuild target: 'hwloc', - directory: 'scons_local', - scm: [url: 'https://github.com/open-mpi/hwloc.git', - branch: "${env.HWLOC_COMMIT}", - cleanAfterCheckout: true], - no_install: true, // No separate install step - TARGET_PREFIX: '/testbin', - target_work: 'testbin', - prebuild: 'pushd ${WORKSPACE}/hwloc;' + - ' ./autogen.sh; popd' - echo "hwloc build succeeded" - stash name: "${env.MY_DOCKERFILE}-hwloc", - includes: 'testbin/hwloc/**' - } // steps - } // stage ('hwloc prebuild') - } // parallel + } + stage ('code_lint_check') { + // Just use CentOS for lint checks. + agent { + dockerfile { + filename "${params.PDOCKERFILE ? params.PDOCKERFILE : 'Dockerfile.centos.7'}" + dir 'scons_local/docker' + label 'docker_runner' + additionalBuildArgs "-t ${sanitized_JOB_NAME}-centos7 " + + '$BUILDARGS' + } + } // agent + steps { + sh "pip freeze || true" + checkPatch review_creds: "${env.GITHUB_CREDS}" + } // steps + } // stage ('code_lint_check') + stage ('fuse build') { + agent { + dockerfile { + filename "${params.PDOCKERFILE ? params.PDOCKERFILE : 'Dockerfile.centos.7'}" + dir 'scons_local/docker' + label 'docker_runner' + additionalBuildArgs "-t ${sanitized_JOB_NAME}-centos7 " + + '$BUILDARGS' + } + } // agent + steps { + // Older scons_local_review only used master branch. + // Newer one looked up the last known good build of master branch. + // Pipeline currently does not have access that info. + sh 'rm -rf testbin/ && mkdir -p testbin' + sconsBuild target: 'fuse', + directory: 'scons_local', + scm: [url: 'https://github.com/libfuse/libfuse.git', + branch: "${env.FUSE_COMMIT}", + cleanAfterCheckout: true], + no_install: true, // No separate install step + TARGET_PREFIX: '/testbin', + target_work: 'testbin' + echo "fuse build succeeded" + } // steps + } // stage ('fuse build') + stage ('openpa prebuild') { + agent { + dockerfile { + filename "${params.PDOCKERFILE ? params.PDOCKERFILE : 'Dockerfile.centos.7'}" + dir 'scons_local/docker' + label 'docker_runner' + additionalBuildArgs "-t ${sanitized_JOB_NAME}-centos7 " + + '$BUILDARGS' + } + } // agent + steps { + sh 'rm -rf testbin/ && mkdir -p testbin' + sconsBuild target: 'openpa', + directory: 'scons_local', + scm: [url: 'http://git.mcs.anl.gov/radix/openpa.git', + branch: "${env.OPENPA_COMMIT}", + cleanAfterCheckout: true], + no_install: true, // No separate install step + TARGET_PREFIX: '/testbin', + target_work: 'testbin' + echo "openpa build succeeded" + stash name: "${env.MY_DOCKERFILE}-openpa", + includes: 'testbin/openpa/**' + } // steps + } // stage ('openpa prebuild') + stage ('hwloc prebuild') { + agent { + dockerfile { + filename "${params.PDOCKERFILE ? params.PDOCKERFILE : 'Dockerfile.centos.7'}" + dir 'scons_local/docker' + label 'docker_runner' + additionalBuildArgs "-t ${sanitized_JOB_NAME}-centos7 " + + '$BUILDARGS' + } + } // agent + steps { + sh 'rm -rf testbin/ && mkdir -p testbin' + sconsBuild target: 'hwloc', + directory: 'scons_local', + scm: [url: 'https://github.com/open-mpi/hwloc.git', + branch: "${env.HWLOC_COMMIT}", + cleanAfterCheckout: true], + no_install: true, // No separate install step + TARGET_PREFIX: '/testbin', + target_work: 'testbin', + prebuild: 'pushd ${WORKSPACE}/hwloc;' + + ' ./autogen.sh; popd' + echo "hwloc build succeeded" + stash name: "${env.MY_DOCKERFILE}-hwloc", + includes: 'testbin/hwloc/**' + } // steps + } // stage ('hwloc prebuild') + } // parallel } // stage ('phase 1') stage('Phase 2') { - parallel { - stage ('CaRT build') { - agent { - dockerfile { - filename "${params.PDOCKERFILE ? params.PDOCKERFILE : 'Dockerfile.centos.7'}" - dir 'scons_local/docker' - label 'docker_runner' - additionalBuildArgs '$BUILDARGS' - } - } // agent - steps { - sconsBuild target: 'cart', - scons_local_replace: true, - scm: [url: 'https://github.com/daos-stack/cart.git', - cleanAfterCheckout: true, - withSubmodules: true] - echo "CaRT build succeeded" - } // steps - } // stage ('CaRT build') - stage ('daos depends') { - agent { - dockerfile { - filename "${params.PDOCKERFILE ? params.PDOCKERFILE : 'Dockerfile.centos.7'}" - dir 'scons_local/docker' - label 'docker_runner' - additionalBuildArgs '$BUILDARGS' + parallel { + stage ('CaRT build with RPMs') { + agent { + dockerfile { + filename 'Dockerfile.centos.7' + dir 'scons_local/docker' + label 'docker_runner' + additionalBuildArgs "-t ${sanitized_JOB_NAME}-centos7 " + + '$BUILDARGS --build-arg USE_RPMS=true ' + + get_deps_build_vars() + } + } // agent + steps { + sconsBuild target: 'cart', + scons_local_replace: true, + scm: [url: 'https://github.com/daos-stack/cart.git', + cleanAfterCheckout: true, + withSubmodules: true], + log_to_file: "scons_out" + echo "CaRT build with RPMs succeeded" + sh 'if egrep \'' + log_timestamp_re + ''' RUN: \' scons_out; then + echo "RUN commands found when none should have been" + exit 1 + fi''' + } // steps + post { + cleanup { + sh 'rm -f scons_out' + } } - } // agent - steps { - // skip cart as we are already buiding that. - sconsBuild target: 'daos', - REQUIRES: 'pmdk,argobots,isal,protobufc', - directory: 'scons_local', - scons_local_replace: true, - scm: [url: 'https://github.com/daos-stack/daos.git', - cleanAfterCheckout: true, - withSubmodules: true] - echo "daos depends build succeeded" - } // steps - } // stage ('daos depends') - stage ('basic checks') { - agent { - dockerfile { - filename "${params.PDOCKERFILE ? params.PDOCKERFILE : 'Dockerfile.centos.7'}" - dir 'scons_local/docker' - label 'docker_runner' - additionalBuildArgs '$BUILDARGS' - } - } // agent - steps { - //checkout scm - // Older scons_local_review only used master branch. - // Newer one looked up the last known good build of master branch. - // Pipeline currently does not have access that info. - checkoutScm url: 'http://git.mcs.anl.gov/radix/openpa.git', - branch: "${env.OPENPA_COMMIT}", - checkoutDir: 'openpa', - cleanAfterCheckout: true - checkoutScm url: 'https://github.com/mercury-hpc/mercury.git', - branch: "${env.MERCURY_COMMIT}", - checkoutDir: 'mercury', - cleanAfterCheckout: true, - withSubmodules: true - checkoutScm url: 'https://github.com/ofiwg/libfabric.git', - branch: "${env.OFI_COMMIT}", - checkoutDir: 'ofi', - cleanAfterCheckout: true - checkoutScm url: 'https://github.com/pmix/master.git', - branch: "${env.PMIX_COMMIT}", - checkoutDir: 'pmix', - cleanAfterCheckout: true - checkoutScm url: 'https://github.com/open-mpi/ompi.git', - branch: "${env.OMPI_COMMIT}", - checkoutDir: 'ompi', - cleanAfterCheckout: true - sh('''find /testbin -maxdepth 1 -type l -print 0 | xargs -r0 rm - rm -rf testbin - for new_dir in openpa hwloc; do - mkdir -p testbin/${new_dir} - ln -s ${PWD}/testbin/${new_dir} /testbin/${new_dir} - done''') - runTest stashes: ["${env.MY_DOCKERFILE}-hwloc", - "${env.MY_DOCKERFILE}-openpa"], - script: '''#!/bin/bash - set -e - export WORKSPACE="" - export prebuilt1="PREBUILT_PREFIX=/testbin/hwloc" - prebuilt1+=":/testbin/openpa" - export prebuilt2="HWLOC_PREBUILT=/testbin/hwloc" - prebuilt2+=" OPENPA_PREBUILT=/testbin/openpa" - export SRC_PREFIX="${PWD}" - SRC_PREFIX+=":${PWD}/scons_local/test/prereq" - pushd scons_local + } + stage ('CaRT build') { + agent { + dockerfile { + filename "${params.PDOCKERFILE ? params.PDOCKERFILE : 'Dockerfile.centos.7'}" + dir 'scons_local/docker' + label 'docker_runner' + additionalBuildArgs "-t ${sanitized_JOB_NAME}-centos7 " + + '$BUILDARGS' + } + } // agent + steps { + sconsBuild target: 'cart', + scons_local_replace: true, + scm: [url: 'https://github.com/daos-stack/cart.git', + cleanAfterCheckout: true, + withSubmodules: true] + echo "CaRT build succeeded" + } // steps + } // stage ('CaRT build') + stage ('daos depends') { + agent { + dockerfile { + filename "${params.PDOCKERFILE ? params.PDOCKERFILE : 'Dockerfile.centos.7'}" + dir 'scons_local/docker' + label 'docker_runner' + additionalBuildArgs "-t ${sanitized_JOB_NAME}-centos7 " + + '$BUILDARGS' + } + } // agent + steps { + // skip cart as we are already buiding that. + sconsBuild target: 'daos', + REQUIRES: 'pmdk,argobots,isal,protobufc', + directory: 'scons_local', + scons_local_replace: true, + scm: [url: 'https://github.com/daos-stack/daos.git', + cleanAfterCheckout: true, + withSubmodules: true] + echo "daos depends build succeeded" + } // steps + } // stage ('daos depends') + stage ('basic checks') { + agent { + dockerfile { + filename "${params.PDOCKERFILE ? params.PDOCKERFILE : 'Dockerfile.centos.7'}" + dir 'scons_local/docker' + label 'docker_runner' + additionalBuildArgs "-t ${sanitized_JOB_NAME}-centos7 " + + '$BUILDARGS' + } + } // agent + steps { + //checkout scm + // Older scons_local_review only used master branch. + // Newer one looked up the last known good build of master branch. + // Pipeline currently does not have access that info. + checkoutScm url: 'http://git.mcs.anl.gov/radix/openpa.git', + branch: "${env.OPENPA_COMMIT}", + checkoutDir: 'openpa', + cleanAfterCheckout: true + checkoutScm url: 'https://github.com/mercury-hpc/mercury.git', + branch: "${env.MERCURY_COMMIT}", + checkoutDir: 'mercury', + cleanAfterCheckout: true, + withSubmodules: true + checkoutScm url: 'https://github.com/ofiwg/libfabric.git', + branch: "${env.OFI_COMMIT}", + checkoutDir: 'ofi', + cleanAfterCheckout: true + checkoutScm url: 'https://github.com/pmix/master.git', + branch: "${env.PMIX_COMMIT}", + checkoutDir: 'pmix', + cleanAfterCheckout: true + checkoutScm url: 'https://github.com/open-mpi/ompi.git', + branch: "${env.OMPI_COMMIT}", + checkoutDir: 'ompi', + cleanAfterCheckout: true + sh('''find /testbin -maxdepth 1 -type l -print0 | xargs -r0 rm + rm -rf testbin + for new_dir in openpa hwloc; do + mkdir -p testbin/${new_dir} + ln -s ${PWD}/testbin/${new_dir} /testbin/${new_dir} + done''') + runTest stashes: ["${env.MY_DOCKERFILE}-hwloc", + "${env.MY_DOCKERFILE}-openpa"], + script: '''#!/bin/bash + set -e + export WORKSPACE="" + export prebuilt1="PREBUILT_PREFIX=/testbin/hwloc" + prebuilt1+=":/testbin/openpa" + export prebuilt2="HWLOC_PREBUILT=/testbin/hwloc" + prebuilt2+=" OPENPA_PREBUILT=/testbin/openpa" + export SRC_PREFIX="${PWD}" + SRC_PREFIX+=":${PWD}/scons_local/test/prereq" + pushd scons_local ./test_scons_local.sh - popd''' - } // steps - } // stage ('basic checks') - } // parallel + popd''' + } // steps + } // stage ('basic checks') + } // parallel } // stage ('phase 2') } // stages } // pipeline diff --git a/components/__init__.py b/components/__init__.py index e0215d9..4bc1e01 100644 --- a/components/__init__.py +++ b/components/__init__.py @@ -21,11 +21,11 @@ # -*- coding: utf-8 -*- """Defines common components used by HPDD projects""" +import os +import platform from prereq_tools import GitRepoRetriever from prereq_tools import WebRetriever from prereq_tools import ProgramBinary -import os -import platform # Check if this is an ARM platform PROCESSOR = platform.machine() @@ -36,6 +36,17 @@ NINJA_PROG = ProgramBinary('ninja', ["ninja-build", "ninja"]) +def inst(reqs, name): + """Return True if name is in list of installed packages""" + set([name, 'all']).intersection(set(reqs.installed)) + +def check(reqs, name, built_str, installed_str=""): + """Return a different string based on whether a component is + installed or not""" + if inst(reqs, name): + return installed_str + return built_str + def define_mercury(reqs): """mercury definitions""" libs = ['rt'] @@ -50,7 +61,8 @@ def define_mercury(reqs): reqs.define('mercury', retriever=retriever, commands=['cmake -DMERCURY_USE_CHECKSUMS=OFF ' - '-DOPA_LIBRARY=$OPENPA_PREFIX/lib/libopa.a ' + '-DOPA_LIBRARY=$OPENPA_PREFIX/lib' + + check(reqs, 'openpa', '', '64') + '/libopa.a ' '-DOPA_INCLUDE_DIR=$OPENPA_PREFIX/include/ ' '-DCMAKE_INSTALL_PREFIX=$MERCURY_PREFIX ' '-DBUILD_EXAMPLES=OFF ' @@ -62,14 +74,16 @@ def define_mercury(reqs): '-DBUILD_DOCUMENTATION=OFF ' '-DBUILD_SHARED_LIBS=ON $MERCURY_SRC ' '-DCMAKE_INSTALL_RPATH=$MERCURY_PREFIX/lib ' - '-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE ' - '-DOFI_INCLUDE_DIR=$OFI_PREFIX/include ' - '-DOFI_LIBRARY=$OFI_PREFIX/lib/libfabric.so' - , 'make $JOBS_OPT', 'make install'], + '-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE ' + + check(reqs, 'ofi', + '-DOFI_INCLUDE_DIR=$OFI_PREFIX/include ' + '-DOFI_LIBRARY=$OFI_PREFIX/lib/libfabric.so'), + 'make $JOBS_OPT', 'make install'], libs=['mercury', 'na', 'mercury_util'], requires=['openpa', 'boost', 'ofi'] + libs, extra_include_path=[os.path.join('include', 'na')], - out_of_src_build=True) + out_of_src_build=True, + package='mercury-devel' if inst(reqs, 'mercury') else None) retriever = GitRepoRetriever('https://github.com/ofiwg/libfabric') reqs.define('ofi', @@ -79,7 +93,8 @@ def define_mercury(reqs): 'make $JOBS_OPT', 'make install'], libs=['fabric'], - headers=['rdma/fabric.h']) + headers=['rdma/fabric.h'], + package='libfabric-devel' if inst(reqs, 'ofi') else None) reqs.define('openpa', retriever=GitRepoRetriever( @@ -87,7 +102,8 @@ def define_mercury(reqs): commands=['$LIBTOOLIZE', './autogen.sh', './configure --prefix=$OPENPA_PREFIX', 'make $JOBS_OPT', - 'make install'], libs=['opa']) + 'make install'], libs=['opa'], + package='openpa-devel' if inst(reqs, 'openpa') else None) if ARM_PLATFORM: url = "https://github.com/mercury-hpc/mchecksum.git" @@ -150,20 +166,21 @@ def define_pmix(reqs): commands=['./configure --prefix=$HWLOC_PREFIX', 'make $JOBS_OPT', 'make install'], headers=['hwloc.h'], - libs=['hwloc']) + libs=['hwloc'], + package='hwloc-devel' if inst(reqs, 'hwloc') else None) retriever = GitRepoRetriever('https://github.com/pmix/master') reqs.define('pmix', retriever=retriever, commands=['./autogen.pl', './configure --with-platform=optimized ' - '--prefix=$PMIX_PREFIX ' - '--with-hwloc=$HWLOC_PREFIX', + '--prefix=$PMIX_PREFIX', 'make $JOBS_OPT', 'make install'], libs=['pmix'], required_progs=['autoreconf', 'aclocal', 'libtool'], headers=['pmix.h'], - requires=['hwloc', 'event']) + requires=['hwloc', 'event'], + package='pmix-devel' if inst(reqs, 'pmix') else None) retriever = GitRepoRetriever('https://github.com/pmix/prrte') @@ -172,9 +189,10 @@ def define_pmix(reqs): commands=['./autogen.pl', './configure --with-platform=optimized ' '--prefix=$PRRTE_PREFIX ' - '--enable-orterun-prefix-by-default ' - '--with-pmix=$PMIX_PREFIX ' - '--with-hwloc=$HWLOC_PREFIX', + '--enable-orterun-prefix-by-default ' + + check(reqs, 'pmix', '--with-pmix=$PMIX_PREFIX') + + ' ' + + check(reqs, 'hwloc', '--with-hwloc=$HWLOC_PREFIX'), 'make $JOBS_OPT', 'make install'], required_progs=['g++', 'flex'], progs=['prun', 'prte', 'prted'], @@ -188,16 +206,21 @@ def define_pmix(reqs): commands=['./autogen.pl --no-oshmem', './configure --with-platform=optimized ' '--enable-orterun-prefix-by-default ' - '--prefix=$OMPI_PREFIX ' - '--with-pmix=$PMIX_PREFIX ' + '--prefix=$OMPI_PREFIX ' + + '--with-pmix=' + + check(reqs, 'pmix', '$PMIX_PREFIX', 'external') + + ' ' + '--disable-mpi-fortran ' '--enable-contrib-no-build=vt ' - '--with-libevent=external ' - '--with-hwloc=$HWLOC_PREFIX', + '--with-libevent=external ' + + '--with-hwloc=' + + check(reqs, 'hwloc', '$HWLOC_PREFIX', + 'external'), 'make $JOBS_OPT', 'make install'], libs=['open-rte'], required_progs=['g++', 'flex'], - requires=['pmix', 'hwloc', 'event']) + requires=['pmix', 'hwloc', 'event'], + package='ompi-devel' if inst(reqs, 'ompi') else None) @@ -250,7 +273,9 @@ def define_components(reqs): "OMPI_PREBUILT=$OMPI_PREFIX " "CART_PREBUILT=$CART_PREFIX " "FUSE_PREBUILT=$FUSE_PREFIX " - "PREFIX=$IOF_PREFIX install"], + "PREFIX=$IOF_PREFIX " + "USE_INSTALLED=" + ','.join(reqs.installed) + ' ' + + "install"], headers=['cnss_plugin.h'], requires=['cart', 'fuse', 'ompi']) @@ -261,7 +286,9 @@ def define_components(reqs): commands=["scons $JOBS_OPT " "OMPI_PREBUILT=$OMPI_PREFIX " "CART_PREBUILT=$CART_PREFIX " - "PREFIX=$DAOS_PREFIX install"], + "PREFIX=$DAOS_PREFIX " + "USE_INSTALLED=" + ','.join(reqs.installed) + ' ' + + "install"], headers=['daos.h'], requires=['cart', 'ompi']) @@ -288,11 +315,14 @@ def define_components(reqs): "OMPI_PREBUILT=$OMPI_PREFIX " "MERCURY_PREBUILT=$MERCURY_PREFIX " "PMIX_PREBUILT=$PMIX_PREFIX " - "PREFIX=$CART_PREFIX install"], + "PREFIX=$CART_PREFIX " + "USE_INSTALLED=" + ','.join(reqs.installed) + ' ' + + "install"], headers=["cart/api.h", "gurt/list.h"], libs=["cart", "gurt"], requires=['mercury', 'uuid', 'crypto', 'ompi', - 'pmix', 'boost', 'yaml']) + 'pmix', 'boost', 'yaml'], + package='cart-devel' if inst(reqs, 'cart') else None) reqs.define('fio', retriever=GitRepoRetriever( diff --git a/docker/Dockerfile.centos.7 b/docker/Dockerfile.centos.7 index 7f93f28..be0ff7d 100755 --- a/docker/Dockerfile.centos.7 +++ b/docker/Dockerfile.centos.7 @@ -1,5 +1,39 @@ +# Copyright (C) 2016-2019 Intel Corporation +# All rights reserved. # -# Copyright 2018, Intel Corporation +# Redistribution and use in source and binary forms, with or without +# modification, are permitted for any purpose (including commercial purposes) +# provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions, and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions, and the following disclaimer in the +# documentation and/or materials provided with the distribution. +# +# 3. In addition, redistributions of modified forms of the source or binary +# code must carry prominent notices stating that the original code was +# changed and the date of the change. +# +# 4. All publications or advertising materials mentioning features or use of +# this software are asked, but not required, to acknowledge that it was +# developed by Intel Corporation and credit the contributors. +# +# 5. Neither the name of Intel Corporation, nor the name of any Contributor +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # 'recipe' for Docker to build an image of centOS-based # environment for building the DAOS project. @@ -9,43 +43,76 @@ FROM centos:7 MAINTAINER Johann Lombardi -# Build arguments can be set via -build-arg -# set NOBUILD to disable git clone & build -ARG NOBUILD +# Build arguments can be set via --build-arg # use same UID as host and default value of 1000 if not specified ARG UID=1000 +ARG USE_RPMS=false -ENV USER daos -ENV PASSWD daos +# for good measure, clean the metadata +RUN yum clean metadata # Install basic tools -RUN yum install -y epel-release yum-plugin-copr; \ - yum -y copr enable jhli/ipmctl; \ - yum -y copr enable jhli/safeclib; \ - yum install -y boost-devel clang-analyzer cmake CUnit-devel doxygen flex \ - gcc gcc-c++ git golang graphviz make lcov meson nasm numactl-devel \ - ninja-build openssl-devel readline-devel pandoc patch scons sg3_utils \ - valgrind valgrind-devel yasm \ - libaio-devel libcmocka libcmocka-devel libevent-devel libibverbs-devel \ - libiscsi-devel librdmacm-devel libtool libtool-ltdl-devel \ - libuuid-devel libyaml-devel \ - python python-jwt python-pep8 python-virtualenv \ - python2-pip python2-pygithub python34-pip python34-pygithub pylint \ - libipmctl-devel - -RUN useradd -u $UID -ms /bin/bash $USER ;echo "$USER:$PASSWD" | chpasswd +RUN yum -y install epel-release +RUN yum -y install \ + git gcc gcc-c++ make cmake golang libtool boost-devel \ + libuuid-devel openssl-devel libevent-devel \ + libtool-ltdl-devel libcmocka-devel readline-devel \ + doxygen pandoc flex patch nasm yasm ninja-build meson \ + CUnit-devel libaio-devel clang-analyzer sg3_utils \ + libiscsi-devel numactl-devel graphviz \ + libyaml-devel \ + scons pylint file ShellCheck python-pep8 lcov python \ + python2-pygithub yum-plugin-copr + +# external deps build/repos +# sadly this doesn't work +#RUN yum-config-manager --save --setopt=*_job_daos-stack_job_${ext}_job_master_lastSuccessfulBuild_artifact_artifacts_.gpgcheck=0 +RUN if ! $USE_RPMS; then exit 0; fi; \ +yum -y install hwloc-devel; \ +for ext in openpa libfabric mercury pmix ompi; do \ + yum-config-manager --add-repo=https://build.hpdd.intel.com/job/daos-stack/job/${ext}/job/master/lastSuccessfulBuild/artifact/artifacts/; \ + echo "gpgcheck = False" >> /etc/yum.repos.d/build.hpdd.intel.com_job_daos-stack_job_${ext}_job_master_lastSuccessfulBuild_artifact_artifacts_.repo; \ +done + +ARG OPENPA=unknown +ARG LIBFABRIC=unknown +ARG MERCURY=unknown +ARG PMIX=unknown +ARG OMPI=unknown + +RUN if ! $USE_RPMS; then exit 0; fi; \ +OPENPA=${OPENPA} \ +LIBFABRIC=${LIBFABRIC} \ +MERCURY=${MERCURY} \ +PMIX=${PMIX} \ +OMPI=${OMPI} \ +yum -y install openpa-devel libfabric-devel mercury-devel pmix-devel ompi-devel + +RUN yum -y copr enable jhli/ipmctl +RUN yum -y copr enable jhli/safeclib +RUN yum -y install libipmctl-devel + +RUN yum -y install librdmacm-devel valgrind valgrind-devel libibverbs-devel \ + python-jwt python-virtualenv python2-pip python34-pip \ + python34-pygithub + +ENV USER daos +ENV PASSWD daos +RUN useradd -u $UID -ms /bin/bash $USER +RUN echo "$USER:$PASSWD" | chpasswd # Create directory for DAOS backend storage RUN mkdir -p /mnt/daos; chown $UID /mnt/daos # Create a target directory for prebuilts RUN mkdir -p /testbin; chown $UID /testbin; ls -ld /testbin - # Switch to new user USER $USER - WORKDIR /home/$USER +# set NOBUILD to disable git clone & build +ARG NOBUILD + RUN if [ -n "${HTTP_PROXY}" ]; then \ git config --global http.proxy "${HTTP_PROXY}"; \ printf "proxy = %s\n" "${HTTP_PROXY##*/}" >> ~/.curlrc; fi diff --git a/prereq_tools/base.py b/prereq_tools/base.py index 3d17781..93d6fb4 100644 --- a/prereq_tools/base.py +++ b/prereq_tools/base.py @@ -1058,6 +1058,12 @@ def require(self, if comp_def.build(env, needed_libs): self.__required[comp] = False changes = True + else: + if comp_def.src_path and \ + not os.path.exists(comp_def.src_path): + self.save_component_prefix('%s_PREFIX' % + comp_def.name.upper(), + "/usr") except Exception as error: # Save the exception in case the component is requested again self.__errors[comp] = error diff --git a/test_scons_local.sh b/test_scons_local.sh index 7900f7c..2e101da 100755 --- a/test_scons_local.sh +++ b/test_scons_local.sh @@ -52,27 +52,20 @@ if [ "$1" != "utest" ]; then python test/validate_build_info.py scons -C test -f SConstruct "${prebuilt2[@]}" --build-deps=yes --config=force python test/validate_build_info.py - set +e - scons -C test -f SConstruct "${prebuilt2[@]}" --build-deps=yes \ - --config=force --require-optional - if [ "${PIPESTATUS[0]}" -eq 0 ]; then - echo "Test for --require-optional failed" - exit 1 + if scons -C test -f SConstruct "${prebuilt2[@]}" --build-deps=yes \ + --config=force --require-optional; then + echo "Test for --require-optional failed" + exit 1 fi - set -e fi -set +e -command -v clang-format >> /dev/null 2>&1 -if [ "${PIPESTATUS[0]}" -eq 0 ]; then - set -e +if command -v clang-format >> /dev/null 2>&1; then directory=$(pwd) site_scons="${directory}/test/tool/site_scons" trap 'rm -f "${site_scons}"' EXIT ln -s "${directory}" "${site_scons}" scons -C test/tool -f SConstruct fi -set -e check_cmd() { @@ -92,13 +85,13 @@ check_cmd() fi fi grep "Valgrind.*check failed" scons_utest_output.txt - result=$? + result=${PIPESTATUS[0]} if [ "$issues" = "clean" ]; then - if [ $result -eq 0 ]; then + if [ "$result" -eq "0" ]; then failed=$((failed + 1)) fi else - if [ $result -ne 0 ]; then + if [ "$result" -ne "0" ]; then failed=$((failed + 1)) fi fi @@ -123,10 +116,10 @@ run_unit_tests() check_cmd 'fail' 'clean' scons -C test -f SConstruct.utest \ --test-name=fail if [ $failed -ne 0 ]; then - echo "Unit test failure" - exit $failed + echo "Unit test failure" + exit $failed else - echo "All unit tests passed" + echo "All unit tests passed" fi }