From 287521aa4584953d2285846b75235e8a29c02d59 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 16 Sep 2026 10:28:53 -0500 Subject: [PATCH 1/9] Shard complete CI and add Open MPI --- .github/workflows/extra.yml | 294 +++++++++++++----- ...niform_sampling_with_variable_resources.py | 2 + .../regression_tests/test_asktell_gpCAM.py | 2 + .../tests/regression_tests/test_gpCAM.py | 2 + .../regression_tests/test_mfkg_branin.py | 2 + .../test_persistent_surmise_calib.py | 2 + .../test_persistent_surmise_killsims.py | 2 + .../test_proxystore_integration.py | 1 + libensemble/tests/run_tests.py | 22 +- .../tests/unit_tests/test_run_tests.py | 4 + pyproject.toml | 32 +- 11 files changed, 268 insertions(+), 97 deletions(-) diff --git a/.github/workflows/extra.yml b/.github/workflows/extra.yml index f3f6fd223..8919a8150 100644 --- a/.github/workflows/extra.yml +++ b/.github/workflows/extra.yml @@ -2,109 +2,237 @@ name: libEnsemble-complete-CI on: workflow_dispatch: + inputs: + run_exhaustive: + description: Run the legacy Python, OS, and transport matrix + required: false + default: false + type: boolean + +concurrency: + group: complete-${{ github.ref }} + cancel-in-progress: true + +env: + HYDRA_LAUNCHER: fork + MPIR_CVAR_CH4_NETMOD: ofi + TERM: xterm-256color + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: - test-libE: - runs-on: ${{ matrix.os }} + core: + runs-on: ubuntu-latest strategy: fail-fast: false matrix: - os: [ubuntu-latest] - mpi-version: [mpich] - python-version: ["py312e", "py313e", "py314e"] - comms-type: [m, l] - include: - - os: macos-latest - python-version: "py312e" - mpi-version: mpich - comms-type: m - - os: macos-latest - python-version: "py312e" - mpi-version: mpich - comms-type: l - - os: ubuntu-latest - mpi-version: openmpi - python-version: "py312e" - comms-type: l - - env: - HYDRA_LAUNCHER: "fork" - MPIR_CVAR_CH4_NETMOD: "ofi" - TERM: xterm-256color - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - defaults: - run: - shell: bash -l {0} - + comms: [m, l] steps: - uses: actions/checkout@v7 with: lfs: true - - - name: Checkout lockfile - run: git lfs checkout - - uses: prefix-dev/setup-pixi@v0.10.2 with: pixi-version: v0.69.0 cache: true frozen: true - environments: ${{ matrix.python-version }} - activate-environment: ${{ matrix.python-version }} - - - name: Install other testing dependencies - run: | - pixi run -e ${{ matrix.python-version }} install/install_minq.sh - - - name: Install libEnsemble, flake8, lock environment - run: | - pip install -e . - flake8 libensemble - - - name: Install gpcam - if: matrix.python-version != 'py313e' && matrix.python-version != 'py314e' - run: | - pixi run -e ${{ matrix.python-version }} pip install gpcam==8.1.13 - - - name: Remove test using octave, gpcam, globus-compute on Python 3.13 - if: matrix.python-version == 'py313e' || matrix.python-version == 'py314e' - run: | - rm ./libensemble/tests/unit_tests/test_ufunc_runners.py # needs globus-compute - rm ./libensemble/tests/regression_tests/test_gpCAM.py # needs gpcam, which doesn't build on 3.13 - rm ./libensemble/tests/regression_tests/test_asktell_gpCAM.py # needs gpcam, which doesn't build on 3.13 - rm ./libensemble/tests/regression_tests/test_optimas_ax_mf.py # needs ax-platform, which doesn't yet support 3.14 - rm ./libensemble/tests/regression_tests/test_optimas_ax_sf.py # needs ax-platform, which doesn't yet support 3.14 - - - name: Start Redis - if: matrix.os == 'ubuntu-latest' - uses: supercharge/redis-github-action@v2 + environments: py312e + - name: Install MINQ + run: pixi run -e py312e install/install_minq.sh + - name: Run core tests + run: >- + pixi run -e py312e ./libensemble/tests/run_tests.py -r -e -${{ matrix.comms }} + --tier core --no-coverage --fail-fast --durations 25 + --timings-json timings-core-${{ matrix.comms }}.json + - name: Upload timings + if: always() + uses: actions/upload-artifact@v4 with: - redis-version: 7 + name: complete-core-${{ matrix.comms }}-timings + path: timings-core-${{ matrix.comms }}.json + if-no-files-found: ignore - - name: Run extensive tests, Ubuntu - if: matrix.os == 'ubuntu-latest' - run: | - ./libensemble/tests/run_tests.py -e -${{ matrix.comms-type }} - - - name: Run extensive tests, macOS - if: matrix.os == 'macos-latest' + external: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [0, 1] + steps: + - uses: actions/checkout@v7 + with: + lfs: true + - uses: prefix-dev/setup-pixi@v0.10.2 + with: + pixi-version: v0.69.0 + cache: true + frozen: true + environments: py312e + - name: Install external test dependencies run: | - pixi run -e ${{ matrix.python-version }} ./libensemble/tests/run_tests.py -e -${{ matrix.comms-type }} + pixi run -e py312e install/install_minq.sh + pixi run -e py312e pip install gpcam==8.1.13 + - name: Run external integration shard + run: >- + pixi run -e py312e ./libensemble/tests/run_tests.py -r -e -l + --tier external --no-coverage --fail-fast --durations 25 + --exclude-match test_proxystore_integration.py + --shard ${{ matrix.shard }}/2 + --timings-json timings-external-${{ matrix.shard }}.json + - name: Upload timings + if: always() + uses: actions/upload-artifact@v4 + with: + name: external-${{ matrix.shard }}-timings + path: timings-external-${{ matrix.shard }}.json + if-no-files-found: ignore - - name: Merge coverage - run: | - mv libensemble/tests/.cov* . + slow: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + lfs: true + - uses: prefix-dev/setup-pixi@v0.10.2 + with: + pixi-version: v0.69.0 + cache: true + frozen: true + environments: py312e + - name: Install slow-test dependencies + run: pixi run -e py312e pip install gpcam==8.1.13 + - name: Run slow integration tests + run: >- + pixi run -e py312e ./libensemble/tests/run_tests.py -r -e -l + --tier slow --no-coverage --fail-fast --durations 25 + --timings-json timings-slow.json + - name: Upload timings + if: always() + uses: actions/upload-artifact@v4 + with: + name: slow-test-timings + path: timings-slow.json + if-no-files-found: ignore - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v7 - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + external-mpi-smoke: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + lfs: true + - uses: prefix-dev/setup-pixi@v0.10.2 + with: + pixi-version: v0.69.0 + cache: true + frozen: true + environments: py312e + - name: Run representative external MPI tests + run: >- + pixi run -e py312e ./libensemble/tests/run_tests.py -r -e -m + --no-coverage --fail-fast --largest-nprocs-only --durations 20 + --match test_mfkg_branin.py + --match test_persistent_aposmm_nlopt.py + --match test_optimas_grid_sample.py + + proxystore: + runs-on: ubuntu-latest + services: + redis: + image: redis:7 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v7 + with: + lfs: true + - uses: prefix-dev/setup-pixi@v0.10.2 + with: + pixi-version: v0.69.0 + cache: true + frozen: true + environments: py312e + - name: Run ProxyStore integration + run: >- + pixi run -e py312e ./libensemble/tests/run_tests.py -r -e -l + --feature redis --no-coverage --fail-fast --durations 10 - spellcheck: - name: Spellcheck release branch - if: contains(github.base_ref, 'develop') + openmpi: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - - uses: crate-ci/typos@v1.50.1 + with: + lfs: true + - uses: prefix-dev/setup-pixi@v0.10.2 + with: + pixi-version: v0.69.0 + cache: true + frozen: true + environments: openmpi + - name: Verify Open MPI environment + run: pixi run -e openmpi mpiexec --version + - name: Run Open MPI compatibility tests + run: >- + pixi run -e openmpi ./libensemble/tests/run_tests.py -r -m + -a=--oversubscribe --no-coverage --fail-fast --durations 20 + --match test_1d_super_simple.py + --match test_mpi_runners.py + --match test_persistent_uniform_sampling_async.py + + macos-platform: + runs-on: macos-latest + steps: + - uses: actions/checkout@v7 + with: + lfs: true + - uses: prefix-dev/setup-pixi@v0.10.2 + with: + pixi-version: v0.69.0 + cache: true + frozen: true + environments: py312e + - name: Run macOS platform tests + run: >- + pixi run -e py312e ./libensemble/tests/run_tests.py -r -l + --no-coverage --fail-fast --durations 20 + --match test_1d_super_simple.py + --match test_executor_simple.py + --match test_persistent_uniform_sampling_async.py + --match test_sim_dirs_per_calc.py + + exhaustive: + if: ${{ inputs.run_exhaustive == true }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + environment: [py312e, py313e, py314e] + comms: [m, l] + include: + - os: macos-latest + environment: py312e + comms: m + - os: macos-latest + environment: py312e + comms: l + steps: + - uses: actions/checkout@v7 + with: + lfs: true + - uses: prefix-dev/setup-pixi@v0.10.2 + with: + pixi-version: v0.69.0 + cache: true + frozen: true + environments: ${{ matrix.environment }} + - name: Install optional dependencies + if: matrix.environment == 'py312e' + run: pixi run -e py312e pip install gpcam==8.1.13 + - name: Run exhaustive suite + run: >- + pixi run -e ${{ matrix.environment }} ./libensemble/tests/run_tests.py + -e -${{ matrix.comms }} --no-coverage --durations 25 diff --git a/libensemble/tests/functionality_tests/test_uniform_sampling_with_variable_resources.py b/libensemble/tests/functionality_tests/test_uniform_sampling_with_variable_resources.py index f0b3fb2de..cf2d6f81a 100644 --- a/libensemble/tests/functionality_tests/test_uniform_sampling_with_variable_resources.py +++ b/libensemble/tests/functionality_tests/test_uniform_sampling_with_variable_resources.py @@ -15,6 +15,8 @@ # TESTSUITE_COMMS: mpi local # TESTSUITE_NPROCS: 4 # TESTSUITE_EXTRA: true +# TESTSUITE_TIER: slow +# TESTSUITE_FEATURES: external resources executor real-launch from multiprocessing import set_start_method diff --git a/libensemble/tests/regression_tests/test_asktell_gpCAM.py b/libensemble/tests/regression_tests/test_asktell_gpCAM.py index ca4ca3ef9..4880e531a 100644 --- a/libensemble/tests/regression_tests/test_asktell_gpCAM.py +++ b/libensemble/tests/regression_tests/test_asktell_gpCAM.py @@ -17,6 +17,8 @@ # TESTSUITE_COMMS: mpi local # TESTSUITE_NPROCS: 4 # TESTSUITE_EXTRA: true +# TESTSUITE_TIER: slow +# TESTSUITE_FEATURES: external gpcam import sys import warnings diff --git a/libensemble/tests/regression_tests/test_gpCAM.py b/libensemble/tests/regression_tests/test_gpCAM.py index cc6b84e3c..251b2dc57 100644 --- a/libensemble/tests/regression_tests/test_gpCAM.py +++ b/libensemble/tests/regression_tests/test_gpCAM.py @@ -22,6 +22,8 @@ # TESTSUITE_COMMS: mpi local # TESTSUITE_NPROCS: 4 # TESTSUITE_EXTRA: true +# TESTSUITE_TIER: slow +# TESTSUITE_FEATURES: external gpcam import sys import warnings diff --git a/libensemble/tests/regression_tests/test_mfkg_branin.py b/libensemble/tests/regression_tests/test_mfkg_branin.py index 9184b534a..c25ed21c7 100644 --- a/libensemble/tests/regression_tests/test_mfkg_branin.py +++ b/libensemble/tests/regression_tests/test_mfkg_branin.py @@ -30,6 +30,8 @@ # TESTSUITE_NPROCS: 4 # TESTSUITE_EXTRA: true # TESTSUITE_OS_SKIP: OSX +# TESTSUITE_TIER: slow +# TESTSUITE_FEATURES: external botorch import numpy as np diff --git a/libensemble/tests/regression_tests/test_persistent_surmise_calib.py b/libensemble/tests/regression_tests/test_persistent_surmise_calib.py index 978ce2b32..b958db57f 100644 --- a/libensemble/tests/regression_tests/test_persistent_surmise_calib.py +++ b/libensemble/tests/regression_tests/test_persistent_surmise_calib.py @@ -26,6 +26,8 @@ # TESTSUITE_NPROCS: 4 # TESTSUITE_EXTRA: true # TESTSUITE_OS_SKIP: OSX +# TESTSUITE_TIER: slow +# TESTSUITE_FEATURES: external surmise # Requires: # Install Surmise package diff --git a/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py b/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py index 0904cf26a..45cc7c51d 100644 --- a/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py +++ b/libensemble/tests/regression_tests/test_persistent_surmise_killsims.py @@ -26,6 +26,8 @@ # TESTSUITE_NPROCS: 4 # TESTSUITE_EXTRA: true # TESTSUITE_OS_SKIP: OSX +# TESTSUITE_TIER: slow +# TESTSUITE_FEATURES: external surmise real-launch # Requires: # Install Surmise package diff --git a/libensemble/tests/regression_tests/test_proxystore_integration.py b/libensemble/tests/regression_tests/test_proxystore_integration.py index 04f5c6d27..7c4129cf6 100644 --- a/libensemble/tests/regression_tests/test_proxystore_integration.py +++ b/libensemble/tests/regression_tests/test_proxystore_integration.py @@ -15,6 +15,7 @@ # TESTSUITE_NPROCS: 4 # TESTSUITE_OS_SKIP: OSX WIN # TESTSUITE_EXTRA: true +# TESTSUITE_FEATURES: external redis from pathlib import Path diff --git a/libensemble/tests/run_tests.py b/libensemble/tests/run_tests.py index 7b0c1c4a0..1a90e162b 100755 --- a/libensemble/tests/run_tests.py +++ b/libensemble/tests/run_tests.py @@ -278,6 +278,10 @@ def parse_test_directives(test_script, largest_nprocs_only=False): raise ValueError(f"Unknown TESTSUITE_OS_SKIP values in {test_script}: {sorted(unknown_os)}") if directives["tier"] not in {"smoke", "core", "external", "slow"}: raise ValueError(f"Unknown TESTSUITE_TIER value in {test_script}: {directives['tier']}") + if directives["extra"] and directives["tier"] == "core": + directives["tier"] = "external" + if directives["extra"] and "external" not in directives["features"]: + directives["features"].append("external") if largest_nprocs_only or REG_RUN_LARGEST_TEST_ONLY: directives["nprocs"] = [directives["nprocs"][-1]] return directives @@ -317,7 +321,10 @@ def skip_test(directives, args, current_os, test_script): return True if args.feature and not set(args.feature).issubset(directives["features"]): return True - if args.match and not any(pattern in os.path.basename(test_script) for pattern in args.match): + test_name = os.path.basename(test_script) + if args.match and not any(pattern in test_name for pattern in args.match): + return True + if args.exclude_match and any(pattern in test_name for pattern in args.exclude_match): return True return False @@ -399,9 +406,19 @@ def parse_arguments(): parser.add_argument("--tier", action="append", choices=["smoke", "core", "external", "slow"]) parser.add_argument("--feature", action="append", help="Require a TESTSUITE_FEATURES value") parser.add_argument("--match", action="append", help="Run standalone tests whose filename contains this value") + parser.add_argument("--exclude-match", action="append", help="Skip tests whose filename contains this value") + parser.add_argument("--shard", metavar="INDEX/TOTAL", help="Run one stable shard of the selected standalone tests") args = parser.parse_args() if args.durations < 0: parser.error("--durations must be non-negative") + if args.shard: + try: + index, total = (int(value) for value in args.shard.split("/", 1)) + except ValueError: + parser.error("--shard must have the form INDEX/TOTAL") + if total < 1 or index < 0 or index >= total: + parser.error("--shard requires TOTAL > 0 and 0 <= INDEX < TOTAL") + args.shard = (index, total) return args @@ -448,6 +465,9 @@ def run_regression_tests(root_dir, python_exec, args, current_os): directives = parse_test_directives(test_script, args.largest_nprocs_only) if not skip_test(directives, args, current_os, test_script): selected_test_files.append(test_script) + if args.shard: + shard_index, shard_total = args.shard + selected_test_files = selected_test_files[shard_index::shard_total] if not list_only and any("test_executor_forces_tutorial" in path for path in selected_test_files): build_forces(root_dir) diff --git a/libensemble/tests/unit_tests/test_run_tests.py b/libensemble/tests/unit_tests/test_run_tests.py index 7626159cd..433f3bb1a 100644 --- a/libensemble/tests/unit_tests/test_run_tests.py +++ b/libensemble/tests/unit_tests/test_run_tests.py @@ -10,6 +10,7 @@ def make_args(**overrides): "a": None, "coverage": True, "e": False, + "exclude_match": None, "feature": None, "match": None, "tier": None, @@ -90,3 +91,6 @@ def test_skip_test_applies_tier_feature_and_name_filters(): assert not run_tests.skip_test(directives, args, "LIN", "/tests/test_proxystore.py") assert run_tests.skip_test(directives, args, "LIN", "/tests/test_other.py") + + args.exclude_match = ["proxystore"] + assert run_tests.skip_test(directives, args, "LIN", "/tests/test_proxystore.py") diff --git a/pyproject.toml b/pyproject.toml index b7607238c..3e9ae4901 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,20 +60,21 @@ libensemble = { path = ".", editable = true } [tool.pixi.environments] default = [] -basic = ["basic"] -extra = ["basic", "extra"] -docs = ["docs", "basic"] +basic = ["basic", "mpich"] +extra = ["basic", "mpich", "extra"] +docs = ["docs", "basic", "mpich"] -dev = ["dev", "basic", "extra", "docs"] +dev = ["dev", "basic", "mpich", "extra", "docs"] # CI environments -py312 = ["py312", "basic"] -py313 = ["py313", "basic"] -py314 = ["py314", "basic"] +py312 = ["py312", "basic", "mpich"] +py313 = ["py313", "basic", "mpich"] +py314 = ["py314", "basic", "mpich"] -py312e = ["py312", "py312e", "basic", "extra"] -py313e = ["py313", "py313e", "basic", "extra"] -py314e = ["py314", "py314e", "basic", "extra"] +py312e = ["py312", "py312e", "basic", "mpich", "extra"] +py313e = ["py313", "py313e", "basic", "mpich", "extra"] +py314e = ["py314", "py314e", "basic", "mpich", "extra"] +openmpi = ["py312", "basic", "openmpi"] # Extra tools for dev environment @@ -85,14 +86,19 @@ black = ">=25.12.0,<26" # Basic dependencies for basic CI [tool.pixi.feature.basic.dependencies] -mypy = ">=1.19.1,<2" -mpi = ">=1.0.1,<2" -mpich = ">=4.3.2,<5" mpi4py = ">=4.1.1,<5" scipy = ">=1.15.2,<2" mpmath = "<=1.3.0" nlopt = ">=2.10.0,<3" +[tool.pixi.feature.mpich.dependencies] +mpi = "1.0.*" +mpich = ">=4.3.2,<5" + +[tool.pixi.feature.openmpi.dependencies] +mpi = "1.0.*" +openmpi = ">=5,<6" + [tool.pixi.feature.basic.pypi-dependencies] ibcdfo = { git = "https://github.com/POptUS/IBCDFO.git", subdirectory = "ibcdfo_pypkg" } From cec6b79f2b73abaae7e313e7c4c79ba89f2cff68 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 16 Sep 2026 13:11:22 -0500 Subject: [PATCH 2/9] Restore mypy in stacked CI environments --- pixi.lock | 4 ++-- pyproject.toml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pixi.lock b/pixi.lock index 88f663598..edce16857 100644 --- a/pixi.lock +++ b/pixi.lock @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:240cd548d94f539439fdc5e78a5967ad78a15251bbda151efd27b2b84c0da794 -size 1130891 +oid sha256:a55f71a0b31c70a15a91c4777b4fba021ee3d0046588c4b5704da09704d626dd +size 1235779 diff --git a/pyproject.toml b/pyproject.toml index 3e9ae4901..6b512bab9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,6 +86,7 @@ black = ">=25.12.0,<26" # Basic dependencies for basic CI [tool.pixi.feature.basic.dependencies] +mypy = ">=1.19.1,<2" mpi4py = ">=4.1.1,<5" scipy = ">=1.15.2,<2" mpmath = "<=1.3.0" From f4b6fc3e43b2d29a4e07f006782bedcf30d63a1b Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 16 Sep 2026 14:42:43 -0500 Subject: [PATCH 3/9] Move IBCDFO POUNDERS tests to external tier --- .../test_persistent_aposmm_ibcdfo_pounders.py | 5 ++++- .../test_persistent_aposmm_ibcdfo_pounders_jax.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_ibcdfo_pounders.py b/libensemble/tests/regression_tests/test_persistent_aposmm_ibcdfo_pounders.py index 619c3633b..11fad588e 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_ibcdfo_pounders.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_ibcdfo_pounders.py @@ -23,6 +23,9 @@ # Do not change these lines - they are parsed by run-tests.sh # TESTSUITE_COMMS: local mpi # TESTSUITE_NPROCS: 3 +# TESTSUITE_EXTRA: true +# TESTSUITE_TIER: external +# TESTSUITE_FEATURES: external ibcdfo pounders real-launch import multiprocessing import sys @@ -122,7 +125,7 @@ def synthetic_beamline_mapping(H, _, sim_specs): alloc_specs = {"alloc_f": alloc_f} - exit_criteria = {"sim_max": 500} + exit_criteria = {"sim_max": 500, "wallclock_max": 300} # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, alloc_specs=alloc_specs, libE_specs=libE_specs) diff --git a/libensemble/tests/regression_tests/test_persistent_aposmm_ibcdfo_pounders_jax.py b/libensemble/tests/regression_tests/test_persistent_aposmm_ibcdfo_pounders_jax.py index 2108182a6..bb12eaa84 100644 --- a/libensemble/tests/regression_tests/test_persistent_aposmm_ibcdfo_pounders_jax.py +++ b/libensemble/tests/regression_tests/test_persistent_aposmm_ibcdfo_pounders_jax.py @@ -23,6 +23,9 @@ # Do not change these lines - they are parsed by run-tests.sh # TESTSUITE_COMMS: local mpi # TESTSUITE_NPROCS: 3 +# TESTSUITE_EXTRA: true +# TESTSUITE_TIER: external +# TESTSUITE_FEATURES: external ibcdfo pounders real-launch import multiprocessing import sys @@ -119,7 +122,7 @@ def synthetic_beamline_mapping(H, _, sim_specs): alloc_specs = {"alloc_f": alloc_f} - exit_criteria = {"sim_max": 500} + exit_criteria = {"sim_max": 500, "wallclock_max": 300} # Perform the run H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, alloc_specs=alloc_specs, libE_specs=libE_specs) From 95a42decaf983e842cef96017227cfaf443781d9 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 16 Sep 2026 15:43:49 -0500 Subject: [PATCH 4/9] Bound external integration shard runtime --- .github/workflows/extra.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/extra.yml b/.github/workflows/extra.yml index 8919a8150..57003b28a 100644 --- a/.github/workflows/extra.yml +++ b/.github/workflows/extra.yml @@ -53,6 +53,7 @@ jobs: external: runs-on: ubuntu-latest + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -73,6 +74,7 @@ jobs: pixi run -e py312e pip install gpcam==8.1.13 - name: Run external integration shard run: >- + timeout --signal=TERM --kill-after=30s 8m pixi run -e py312e ./libensemble/tests/run_tests.py -r -e -l --tier external --no-coverage --fail-fast --durations 25 --exclude-match test_proxystore_integration.py From af199eea475bc6f30984972ed14869b2de608676 Mon Sep 17 00:00:00 2001 From: jlnav Date: Wed, 16 Sep 2026 15:52:52 -0500 Subject: [PATCH 5/9] Prevent APOSMM optimizer startup deadlocks --- libensemble/gen_funcs/aposmm_localopt_support.py | 4 +++- pixi.lock | 4 ++-- pyproject.toml | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/libensemble/gen_funcs/aposmm_localopt_support.py b/libensemble/gen_funcs/aposmm_localopt_support.py index 5f0bd67e6..580d1373c 100644 --- a/libensemble/gen_funcs/aposmm_localopt_support.py +++ b/libensemble/gen_funcs/aposmm_localopt_support.py @@ -654,7 +654,9 @@ def run_local_tao(user_specs, comm_queue, x0, f0, child_can_read, parent_can_rea def opt_runner(run_local_opt, user_specs, comm_queue, x0, f0, child_can_read, parent_can_read): try: run_local_opt(user_specs, comm_queue, x0, f0, child_can_read, parent_can_read) - except Exception: + except BaseException: + # A dependency may call sys.exit() during optimizer startup. Always unblock + # the parent so it can surface that failure instead of waiting indefinitely. comm_queue.put(ErrorMsg(traceback.format_exc())) parent_can_read.set() diff --git a/pixi.lock b/pixi.lock index edce16857..fd764a020 100644 --- a/pixi.lock +++ b/pixi.lock @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a55f71a0b31c70a15a91c4777b4fba021ee3d0046588c4b5704da09704d626dd -size 1235779 +oid sha256:f38f0333fc700edb1bc17f8326ed3f12e212b72196d07268a1699ec1f1cf1571 +size 1230774 diff --git a/pyproject.toml b/pyproject.toml index 6b512bab9..34a501cf7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,7 +101,7 @@ mpi = "1.0.*" openmpi = ">=5,<6" [tool.pixi.feature.basic.pypi-dependencies] -ibcdfo = { git = "https://github.com/POptUS/IBCDFO.git", subdirectory = "ibcdfo_pypkg" } +ibcdfo = { git = "https://github.com/POptUS/IBCDFO.git", rev = "538a7adf690dfd08b410a7567fa114c8fa9783f4", subdirectory = "ibcdfo_pypkg" } # "dev" dependencies needed for basic CI flake8 = ">=7.3.0,<8" From 38a8c3b16288bd312095e2716f9b41b170f1cb68 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 17 Sep 2026 10:20:34 -0500 Subject: [PATCH 6/9] try enabling mpi on coverage job --- .github/workflows/basic.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index 0a7da2686..fda29930c 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -130,7 +130,7 @@ jobs: - name: Install MINQ run: pixi run -e py312 ./install/install_minq.sh - name: Run canonical coverage suite - run: pixi run -e py312 ./libensemble/tests/run_tests.py -l --coverage --fail-fast --durations 25 + run: pixi run -e py312 ./libensemble/tests/run_tests.py -l -m --coverage --fail-fast --durations 25 - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v7 env: From c39bffc46a73a9b585fb108410aa11b7532ddf3e Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 17 Sep 2026 10:32:50 -0500 Subject: [PATCH 7/9] move coverage job into extra.yml. remove coverage for a handful of allocation functions and toy sims --- .github/workflows/basic.yml | 22 ---------------------- .github/workflows/extra.yml | 22 ++++++++++++++++++++++ libensemble/tests/.coveragerc | 12 ++++++++++++ 3 files changed, 34 insertions(+), 22 deletions(-) diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index fda29930c..d8b88188e 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -113,25 +113,3 @@ jobs: --no-coverage --fail-fast --durations 20 --match test_1d_super_simple.py --match test_persistent_uniform_sampling_async.py - - coverage: - if: "! github.event.pull_request.draft" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - with: - lfs: true - - uses: prefix-dev/setup-pixi@v0.10.2 - with: - pixi-version: v0.69.0 - cache: true - frozen: true - environments: py312 - - name: Install MINQ - run: pixi run -e py312 ./install/install_minq.sh - - name: Run canonical coverage suite - run: pixi run -e py312 ./libensemble/tests/run_tests.py -l -m --coverage --fail-fast --durations 25 - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v7 - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/extra.yml b/.github/workflows/extra.yml index 57003b28a..4bdb91255 100644 --- a/.github/workflows/extra.yml +++ b/.github/workflows/extra.yml @@ -51,6 +51,28 @@ jobs: path: timings-core-${{ matrix.comms }}.json if-no-files-found: ignore + coverage: + if: "! github.event.pull_request.draft" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + lfs: true + - uses: prefix-dev/setup-pixi@v0.10.2 + with: + pixi-version: v0.69.0 + cache: true + frozen: true + environments: py312e + - name: Install MINQ + run: pixi run -e py312e ./install/install_minq.sh + - name: Run canonical coverage suite + run: pixi run -e py312e ./libensemble/tests/run_tests.py -l -m --coverage --fail-fast --durations 25 + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v7 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + external: runs-on: ubuntu-latest timeout-minutes: 10 diff --git a/libensemble/tests/.coveragerc b/libensemble/tests/.coveragerc index cd38c7cd3..75599b2d1 100644 --- a/libensemble/tests/.coveragerc +++ b/libensemble/tests/.coveragerc @@ -18,8 +18,20 @@ omit = */unit_tests_nompi/* */unit_tests_logger/* */regression_tests/* + */alloc_funcs/start_persistent_local_opt_gens.py + */alloc_funcs/fast_alloc_and_pausing.py + */alloc_funcs/start_fd_persistent.py */sim_funcs/helloworld.py */sim_funcs/executor_hworld.py + */sim_funcs/chwirut1.py + */sim_funcs/var_resources.py + */sim_funcs/rosenbrock.py + */sim_funcs/borehole_kills.py + */sim_funcs/noisy_vector_mapping.py + */sim_funcs/augmented_branin.py + */sim_funcs/comms_testing.py + */sim_funcs/inverse_bayes.py + */sim_funcs/borehole.py */tools/test_support.py */tools/forkable_pdb.py */tools/parse_args.py From 25697ae132577b2b916a343169b58abb0d626686 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 17 Sep 2026 11:06:15 -0500 Subject: [PATCH 8/9] nonconflicting ensemble dirs --- .../regression_tests/test_GPU_variable_resources_multi_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libensemble/tests/regression_tests/test_GPU_variable_resources_multi_task.py b/libensemble/tests/regression_tests/test_GPU_variable_resources_multi_task.py index 5564e7f96..360a3194a 100644 --- a/libensemble/tests/regression_tests/test_GPU_variable_resources_multi_task.py +++ b/libensemble/tests/regression_tests/test_GPU_variable_resources_multi_task.py @@ -63,7 +63,7 @@ num_resource_sets=gpu_test.nworkers - 1, resource_info={"cores_on_node": (32, 64), "gpus_on_node": 4}, sim_dirs_make=True, - ensemble_dir_path="./ensemble_GPU_variable_multi_task_w" + str(nworkers), + ensemble_dir_path="./ensemble_GPU_variable_multi_task_w" + str(nworkers) + str(gpu_test.libE_specs.comms), ) gpu_test.sim_specs = SimSpecs( From f38e78259fe25bdd016fbab4a3ce1d24fdcba598 Mon Sep 17 00:00:00 2001 From: jlnav Date: Thu, 17 Sep 2026 11:53:43 -0500 Subject: [PATCH 9/9] include extra tests in coverage jobs --- .github/workflows/extra.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/extra.yml b/.github/workflows/extra.yml index 4bdb91255..13071d638 100644 --- a/.github/workflows/extra.yml +++ b/.github/workflows/extra.yml @@ -67,7 +67,7 @@ jobs: - name: Install MINQ run: pixi run -e py312e ./install/install_minq.sh - name: Run canonical coverage suite - run: pixi run -e py312e ./libensemble/tests/run_tests.py -l -m --coverage --fail-fast --durations 25 + run: pixi run -e py312e ./libensemble/tests/run_tests.py -l -m -e --coverage --fail-fast --durations 25 - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v7 env: