From ac820e0cdd023c7dcbe055e3f9f82be1f6bb2a04 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Tue, 7 Jul 2026 13:36:23 +0200 Subject: [PATCH 1/2] do not show warning for balance issues on cpus --- src/timeIntegrator.cpp | 76 +++++++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/src/timeIntegrator.cpp b/src/timeIntegrator.cpp index 6ffb6acfb..786cb5e0a 100644 --- a/src/timeIntegrator.cpp +++ b/src/timeIntegrator.cpp @@ -197,47 +197,53 @@ double TimeIntegrator::ComputeBalance() { // Check MPI imbalance double imbalance = 0; #ifdef WITH_MPI - const double allowedImbalance = 20.0; - std::vector computeLogPerCore(idfx::psize); - MPI_Gather(&computeLastLog, 1, MPI_DOUBLE, computeLogPerCore.data(), 1, MPI_DOUBLE, 0, - MPI_COMM_WORLD); - computeLastLog = 0; // reset timer for all cores - if(idfx::prank==0) { - // Compute the average, the min and the max - double computeMin = computeLogPerCore[0]; - double computeMax = computeLogPerCore[0]; - double computeMean = 0; - - for(int i = 0 ; i < idfx::psize ; i++) { - computeMean += computeLogPerCore[i]; - if(computeLogPerCore[i]>computeMax) { - computeMax = computeLogPerCore[i]; - } - if(computeLogPerCore[i] computeLogPerCore(idfx::psize); + MPI_Gather(&computeLastLog, 1, MPI_DOUBLE, computeLogPerCore.data(), 1, MPI_DOUBLE, 0, + MPI_COMM_WORLD); + computeLastLog = 0; // reset timer for all cores + if(idfx::prank==0) { + // Compute the average, the min and the max + double computeMin = computeLogPerCore[0]; + double computeMax = computeLogPerCore[0]; + double computeMean = 0; - if(imbalance>allowedImbalance ) { - idfx::cout << "-------------------------------------------------------------"<< std::endl; - idfx::cout << "Warning: MPI imbalance found in this run " << std::endl; - idfx::cout << std::fixed; for(int i = 0 ; i < idfx::psize ; i++) { - if(computeLogPerCore[i]/computeMean - 1> allowedImbalance/2/100) { - idfx::cout << "+" << 100*(computeLogPerCore[i]/computeMean-1) - << "% (proc " << i << ")" << std::endl; + computeMean += computeLogPerCore[i]; + if(computeLogPerCore[i]>computeMax) { + computeMax = computeLogPerCore[i]; } - if(1-computeLogPerCore[i]/computeMean > allowedImbalance/2/100) { - idfx::cout << "-" << 100*(1-computeLogPerCore[i]/computeMean) - << "% (proc " << i << ")" << std::endl; + if(computeLogPerCore[i]allowedImbalance ) { + idfx::cout << "-------------------------------------------------------------" + << std::endl; + idfx::cout << "Warning: MPI imbalance found in this run " << std::endl; + idfx::cout << std::fixed; + for(int i = 0 ; i < idfx::psize ; i++) { + if(computeLogPerCore[i]/computeMean - 1> allowedImbalance/2/100) { + idfx::cout << "+" << 100*(computeLogPerCore[i]/computeMean-1) + << "% (proc " << i << ")" << std::endl; + } + if(1-computeLogPerCore[i]/computeMean > allowedImbalance/2/100) { + idfx::cout << "-" << 100*(1-computeLogPerCore[i]/computeMean) + << "% (proc " << i << ")" << std::endl; + } + } + idfx::cout << "You should probably check these nodes are running properly." + << std::endl; + idfx::cout << "-------------------------------------------------------------" + << std::endl; + } } - } + #endif #endif return(imbalance); } From 86d6c297f62b65da47b53318cc760a2558a51b83 Mon Sep 17 00:00:00 2001 From: Geoffroy Lesur Date: Wed, 8 Jul 2026 09:19:45 +0200 Subject: [PATCH 2/2] compute balance on cpus, but show warning only on gpus --- src/timeIntegrator.cpp | 57 +++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/src/timeIntegrator.cpp b/src/timeIntegrator.cpp index 786cb5e0a..7079700d9 100644 --- a/src/timeIntegrator.cpp +++ b/src/timeIntegrator.cpp @@ -197,34 +197,33 @@ double TimeIntegrator::ComputeBalance() { // Check MPI imbalance double imbalance = 0; #ifdef WITH_MPI - // only do this on GPUs - #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) || defined(KOKKOS_ENABLE_SYCL) - const double allowedImbalance = 20.0; - std::vector computeLogPerCore(idfx::psize); - MPI_Gather(&computeLastLog, 1, MPI_DOUBLE, computeLogPerCore.data(), 1, MPI_DOUBLE, 0, - MPI_COMM_WORLD); - computeLastLog = 0; // reset timer for all cores - if(idfx::prank==0) { - // Compute the average, the min and the max - double computeMin = computeLogPerCore[0]; - double computeMax = computeLogPerCore[0]; - double computeMean = 0; - - for(int i = 0 ; i < idfx::psize ; i++) { - computeMean += computeLogPerCore[i]; - if(computeLogPerCore[i]>computeMax) { - computeMax = computeLogPerCore[i]; - } - if(computeLogPerCore[i] computeLogPerCore(idfx::psize); + MPI_Gather(&computeLastLog, 1, MPI_DOUBLE, computeLogPerCore.data(), 1, MPI_DOUBLE, 0, + MPI_COMM_WORLD); + computeLastLog = 0; // reset timer for all cores + if(idfx::prank==0) { + // Compute the average, the min and the max + double computeMin = computeLogPerCore[0]; + double computeMax = computeLogPerCore[0]; + double computeMean = 0; + + for(int i = 0 ; i < idfx::psize ; i++) { + computeMean += computeLogPerCore[i]; + if(computeLogPerCore[i]>computeMax) { + computeMax = computeLogPerCore[i]; } - computeMean /= idfx::psize; - imbalance = (computeMax-computeMin)/computeMean*100; - + if(computeLogPerCore[i]allowedImbalance ) { idfx::cout << "-------------------------------------------------------------" - << std::endl; + << std::endl; idfx::cout << "Warning: MPI imbalance found in this run " << std::endl; idfx::cout << std::fixed; for(int i = 0 ; i < idfx::psize ; i++) { @@ -238,12 +237,12 @@ double TimeIntegrator::ComputeBalance() { } } idfx::cout << "You should probably check these nodes are running properly." - << std::endl; + << std::endl; idfx::cout << "-------------------------------------------------------------" - << std::endl; + << std::endl; } - } - #endif + #endif + } #endif return(imbalance); }