From 7b5eaf104168ade0e42e2244e690ffddc85d4813 Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Thu, 12 Sep 2024 09:53:29 +0200 Subject: [PATCH 01/30] PRE-RELEASE: grace/fs support update to version 2024.7.4 with support for grace/fs --- .version | 2 +- CMakeLists.txt | 23 +- bin/pace_activeset | 21 +- lib/ace/CMakeLists.txt | 13 +- .../src/ace-evaluator/ace_complex.h | 34 + .../src/ace-evaluator/ace_contigous_array.h | 28 +- .../src/ace-evaluator/ace_evaluator.cpp | 8 +- .../src/ace-evaluator/ace_evaluator.h | 6 +- .../src/ace-evaluator/ace_radial.cpp | 12 +- .../src/ace-evaluator/ace_radial.h | 5 + .../src/ace-evaluator/ace_recursive.cpp | 6 +- .../src/ace-evaluator/ace_spherical_cart.cpp | 26 + .../src/ace-evaluator/ace_spherical_cart.h | 4 + .../src/ace-evaluator/ace_version.h | 6 +- .../src/extra/ace_calculator.cpp | 17 +- lib/ace/src/ace/ace_b_evaluator.cpp | 8 +- lib/ace/src/ace/grace_fs_evaluator.cpp | 870 ++++++++++++++++++ lib/ace/src/ace/grace_fs_evaluator.h | 308 +++++++ lib/ace/src/extra/grace_fs_calculator.cpp | 142 +++ lib/ace/src/extra/grace_fs_calculator.h | 42 + .../potentials/FS_model_shift_scale.asi | Bin 0 -> 2116 bytes .../potentials/FS_model_shift_scale.yaml | 41 + setup.py | 7 +- src/pyace/ace/grace_fs_binding.cpp | 101 ++ src/pyace/activelearning.py | 46 +- src/pyace/asecalc.py | 126 +++ src/pyace/data_aug.py | 2 + src/pyace/fitadapter.py | 32 +- tests/test_calculator.py | 74 +- 29 files changed, 1936 insertions(+), 74 deletions(-) create mode 100644 lib/ace/src/ace/grace_fs_evaluator.cpp create mode 100644 lib/ace/src/ace/grace_fs_evaluator.h create mode 100644 lib/ace/src/extra/grace_fs_calculator.cpp create mode 100644 lib/ace/src/extra/grace_fs_calculator.h create mode 100644 lib/ace/test/fitting/potentials/FS_model_shift_scale.asi create mode 100644 lib/ace/test/fitting/potentials/FS_model_shift_scale.yaml create mode 100644 src/pyace/ace/grace_fs_binding.cpp diff --git a/.version b/.version index 0d91a54..1d0ba9e 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -0.3.0 +0.4.0 diff --git a/CMakeLists.txt b/CMakeLists.txt index 6bfcc23..e2d5b68 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,6 +23,7 @@ endif() set(SOURCE_DIR "src/pyace") set(ACE_ROOT "lib/ace") set(ACE_SRC ${ACE_ROOT}/src/ace) +set(ACE_EXTRA_SRC ${ACE_ROOT}/src/extra) set(ACE_INCLUDE ${ACE_ROOT}/src) set(ACE_EVALUATOR_ROOT ${ACE_ROOT}/ace-evaluator) @@ -125,13 +126,22 @@ set(SOURCES_CALCULATOR "${ACE_EVALUATOR_SRC}/ace_evaluator.cpp" "${SOURCE_DIR}/utils/ace_calculator_binding.cpp") +set(GRACE_FS_SOURCES + "${ACE_EVALUATOR_SRC}/ace_abstract_basis.cpp" + "${ACE_EVALUATOR_SRC}/ace_radial.cpp" + "${ACE_EVALUATOR_SRC}/ace_spherical_cart.cpp" + "${SOURCE_DIR}/ace/grace_fs_binding.cpp" + "${ACE_SRC}/grace_fs_evaluator.cpp" + "${ACE_EXTRA_SRC}/grace_fs_calculator.cpp" +) + # C++ FLAGS #--------------------------------------------------------- #set(CMAKE_CXX_FLAGS "-Wall -Wextra") #set(CMAKE_CXX_FLAGS_DEBUG "-g") -set(CMAKE_CXX_FLAGS_RELEASE "-Ofast") # -DNDEBUG -#set(CMAKE_CXX_FLAGS_DEBUG "-Ofast -DNDEBUG") +set(CMAKE_CXX_FLAGS_RELEASE "-Ofast -DNDEBUG") +set(CMAKE_CXX_FLAGS_DEBUG "-Ofast -DNDEBUG") #Finally create the package #-------------------------------------------------------- @@ -165,6 +175,15 @@ if (CMAKE_COMPILER_IS_GNUCC) target_compile_options(evaluator PRIVATE -Wunused-variable -Wvla -Wno-effc++ -Werror=vla) endif () + + +pybind11_add_module(grace_fs ${GRACE_FS_SOURCES}) +target_link_libraries(grace_fs PRIVATE yaml-cpp-pace) +target_link_libraries(grace_fs PRIVATE cnpy) +if (CMAKE_COMPILER_IS_GNUCC) + target_compile_options(grace_fs PRIVATE -Wunused-variable -Wvla -Wno-effc++ -Werror=vla) +endif () + pybind11_add_module(calculator ${SOURCES_CALCULATOR} ) if (CMAKE_COMPILER_IS_GNUCC) target_compile_options(calculator PRIVATE -Wunused-variable -Wvla -Wno-effc++ -Werror=vla) diff --git a/bin/pace_activeset b/bin/pace_activeset index ab3eae1..8797bbc 100644 --- a/bin/pace_activeset +++ b/bin/pace_activeset @@ -20,7 +20,7 @@ from pyace.activelearning import compute_B_projections, compute_active_set, comp from pyace.preparedata import sizeof_fmt from pyace.aceselect import compute_mem_limit, compute_batch_size, compute_required_memory, load_datasets - +from pyace.grace_fs import GRACEFSBasisSet def build_parser(): parser = argparse.ArgumentParser(prog="pace_activeset", @@ -101,9 +101,14 @@ def main(): if verbose: log.info("Potential file: {}".format(potential_file)) - bconf = BBasisConfiguration(potential_file) + try: + bconf = BBasisConfiguration(potential_file) + bbasis = ACEBBasisSet(bconf) + log.info("ACEBBasisSet loaded") + except RuntimeError: + bbasis = GRACEFSBasisSet(potential_file) + log.info("GRACEFSBasisSet loaded") - bbasis = ACEBBasisSet(bconf) nfuncs = compute_number_of_functions(bbasis) if is_full: n_projections = [p * bbasis.map_embedding_specifications[st].ndensity for st, p in enumerate(nfuncs)] @@ -207,21 +212,21 @@ def main(): sel_mask_dict = {} for symb, st in elements_to_index_map.items(): dforces = ref_forces_dict[st] - forces_dict[st] - dforces_norm = np.linalg.norm(dforces, axis=1) + dforces_comp = np.linalg.norm(dforces, axis=1)/np.sqrt(3) if error_based_threshold <= 0: - q1, q2, q3 = np.quantile(dforces_norm, q=[0.25, 0.5, 0.75], ) + q1, q2, q3 = np.quantile(dforces_comp, q=[0.25, 0.5, 0.75], ) iqr = q3 - q1 upper_bound = q3 + 1.5 * iqr else: upper_bound = error_based_threshold - sel_mask = dforces_norm <= upper_bound + sel_mask = dforces_comp <= upper_bound sel_mask_dict[st] = sel_mask - f_rmse_before = np.sqrt(np.mean(dforces_norm ** 2)) + f_rmse_before = np.sqrt(np.mean(dforces_comp ** 2)) - f_rmse_after = np.sqrt(np.mean(dforces_norm[sel_mask] ** 2)) + f_rmse_after = np.sqrt(np.mean(dforces_comp[sel_mask] ** 2)) if verbose: log.info("\tElement: {}, specie type: {}".format(symb, st)) diff --git a/lib/ace/CMakeLists.txt b/lib/ace/CMakeLists.txt index 67cfc9b..3a5f452 100644 --- a/lib/ace/CMakeLists.txt +++ b/lib/ace/CMakeLists.txt @@ -10,6 +10,10 @@ set(FITTING_PATH src/ace) FILE(GLOB FITTING_SOURCE_FILES ${FITTING_PATH}/*.cpp) set(FITTING_INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/src) +set(ACE_EXTRA_PATH src/extra) +FILE(GLOB ACE_EXTRA_SOURCE_FILES ${ACE_EXTRA_PATH}/*.cpp) +set(ACE_EXTRA_INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/src) + FILE(GLOB UTILS_SOURCE_FILES src/utils/*.cpp) set(UTILS_INCLUDE_DIR src/utils) FILE(GLOB TEST_FITTING test/fitting/test_*.cpp) @@ -32,13 +36,14 @@ add_subdirectory(${CNPY_PATH}) ##### test_extend ##### if(DEFINED BUILD_TESTS) add_executable(test_extended ${EVALUATOR_SOURCE_FILES} ${EVALUATOR_EXTRA_FILES} ${EVALUATOR_EXTRA_UTILS_FILES} - ${TEST_FITTING} ${TEST_UTILS} ${FITTING_SOURCE_FILES} ${UTILS_SOURCE_FILES}) + ${TEST_FITTING} ${TEST_UTILS} ${FITTING_SOURCE_FILES} ${UTILS_SOURCE_FILES} ${ACE_EXTRA_SOURCE_FILES}) if (CMAKE_COMPILER_IS_GNUCC) target_compile_options(test_extended PRIVATE -Wunused-variable -Wvla -Wno-effc++ -Werror=vla) endif () target_include_directories(test_extended PUBLIC ${TEST_DIR}) target_include_directories(test_extended PUBLIC ${EVALUATOR_INCLUDE_DIR} ${EVALUATOR_EXTRA_INCLUDE_DIR} ${EVALUATOR_EXTRA_UTILS_INCLUDE_DIR}) target_include_directories(test_extended PUBLIC ${FITTING_INCLUDE_DIR}) + target_include_directories(test_extended PUBLIC ${ACE_EXTRA_INCLUDE_DIR}) target_include_directories(test_extended PUBLIC ${UTILS_INCLUDE_DIR}) #Wigner target_include_directories(test_extended PUBLIC ${WIGNER_INCLUDE_PATH}) @@ -57,7 +62,11 @@ if(DEFINED BUILD_TESTS) target_compile_definitions(test_extended PUBLIC EXTRA_C_PROJECTIONS) target_compile_definitions(test_extended PUBLIC COMPUTE_B_GRAD) target_compile_definitions(test_extended PUBLIC MULTIARRAY_INDICES_CHECK) # - #target_compile_definitions(test_extended PUBLIC FINE_TIMING) # enforce to check the indices range of multiarrays + + # uncomment for performance tests +# target_compile_definitions(test_extended PUBLIC FINE_TIMING) # enforce to check the indices range of multiarrays +# target_compile_definitions(test_extended PUBLIC PERF_TEST) # enforce to check the indices range of multiarrays + #target_compile_definitions(test_extended PUBLIC PRINT_BASIS_FUNCTION) # print B and C-tilde basis functions #target_compile_definitions(test_extended PUBLIC PRINT_MAIN_STEPS) # print main steps (loop over atoms, energy calc, force calc, etc.) #target_compile_definitions(test_extended PUBLIC PRINT_INTERMEDIATE_VALUES) # print intermediate energies and forces values diff --git a/lib/ace/ace-evaluator/src/ace-evaluator/ace_complex.h b/lib/ace/ace-evaluator/src/ace-evaluator/ace_complex.h index 328def0..5d157ee 100644 --- a/lib/ace/ace-evaluator/src/ace-evaluator/ace_complex.h +++ b/lib/ace/ace-evaluator/src/ace-evaluator/ace_complex.h @@ -197,6 +197,21 @@ inline ACEComplex operator+(const DOUBLE_TYPE &real, const ACEComplex &cm) { return cm + real; } +struct ACEDRealYcomponent { +public: + DOUBLE_TYPE a[3]; + + + ACEDRealYcomponent operator*(const DOUBLE_TYPE &rhs) const { + ACEDRealYcomponent res{0}; + res.a[0] = this->a[0] * rhs; + res.a[1] = this->a[1] * rhs; + res.a[2] = this->a[2] * rhs; + return res; + } + +}; + /** A structure to store the derivative of \f$ Y_{lm} \f$ */ @@ -260,6 +275,25 @@ struct ACEDYcomponent { this->a[2].conjugate(); } + ACEDRealYcomponent real() const { + ACEDRealYcomponent res; + res.a[0] = this->a[0].real; + res.a[1] = this->a[1].real; + res.a[2] = this->a[2].real; + + return res; + } + + ACEDRealYcomponent img() const { + ACEDRealYcomponent res; + res.a[0] = this->a[0].img; + res.a[1] = this->a[1].img; + res.a[2] = this->a[2].img; + + return res; + } + + }; diff --git a/lib/ace/ace-evaluator/src/ace-evaluator/ace_contigous_array.h b/lib/ace/ace-evaluator/src/ace-evaluator/ace_contigous_array.h index edbe19d..4606c31 100644 --- a/lib/ace/ace-evaluator/src/ace-evaluator/ace_contigous_array.h +++ b/lib/ace/ace-evaluator/src/ace-evaluator/ace_contigous_array.h @@ -187,7 +187,7 @@ class ContiguousArrayND { * Get array data pointer * @return data array pointer */ - inline T* get_data() const { + inline T *get_data() const { return data; } @@ -243,6 +243,32 @@ class ContiguousArrayND { return is_proxy_; } + /** + * move all data from src into this, starting from dest_ind position + * src array become proxy + * @param src + * @param dest_ind + */ + void move_from(ContiguousArrayND src, size_t dest_ind) { + // check that src is not proxy + if (src.is_proxy()) + throw invalid_argument("src Array1D cannot be proxy when move to another"); + // check that there is enough space + if (dest_ind + src.get_size() > this->size) + throw invalid_argument("Not enough space to move Array1D into another Array1D"); + // copy data + for (size_t i = 0; i < src.get_size(); i++) + this->data[dest_ind + i] = src.data[i]; + + // clean src data + delete[] src.data; + + // change pointer + src.data = &this->data[dest_ind]; + + //set proxy flag + src.is_proxy_ = true; + } }; diff --git a/lib/ace/ace-evaluator/src/ace-evaluator/ace_evaluator.cpp b/lib/ace/ace-evaluator/src/ace-evaluator/ace_evaluator.cpp index d4acd04c..629e815 100644 --- a/lib/ace/ace-evaluator/src/ace-evaluator/ace_evaluator.cpp +++ b/lib/ace/ace-evaluator/src/ace-evaluator/ace_evaluator.cpp @@ -41,12 +41,14 @@ void ACEEvaluator::init(ACEAbstractBasisSet *basis_set) { } void ACEEvaluator::init_timers() { - loop_over_neighbour_timer.init(); + A_construction_timer.init(); forces_calc_loop_timer.init(); forces_calc_neighbour_timer.init(); energy_calc_timer.init(); per_atom_calc_timer.init(); total_time_calc_timer.init(); + setup_timer.init(); + weights_and_theta_timer.init(); } //================================================================================================================ @@ -257,7 +259,7 @@ ACECTildeEvaluator::compute_atom(int i, DOUBLE_TYPE **x, const SPECIES_TYPE *typ const Array1D &gr = basis_set->radial_functions->gr; const Array1D &dgr = basis_set->radial_functions->dgr; - loop_over_neighbour_timer.start(); + A_construction_timer.start(); int jj_actual = 0; SPECIES_TYPE type_j = 0; @@ -377,7 +379,7 @@ ACECTildeEvaluator::compute_atom(int i, DOUBLE_TYPE **x, const SPECIES_TYPE *typ } } } //now A's are constructed - loop_over_neighbour_timer.stop(); + A_construction_timer.stop(); // ==================== ENERGY ==================== diff --git a/lib/ace/ace-evaluator/src/ace-evaluator/ace_evaluator.h b/lib/ace/ace-evaluator/src/ace-evaluator/ace_evaluator.h index 8278f83..33e50c7 100644 --- a/lib/ace/ace-evaluator/src/ace-evaluator/ace_evaluator.h +++ b/lib/ace/ace-evaluator/src/ace-evaluator/ace_evaluator.h @@ -64,13 +64,13 @@ class ACEEvaluator { // set of timers for code profiling - ACETimer loop_over_neighbour_timer; ///< timer for loop over neighbours when constructing A's for single central atom + ACETimer A_construction_timer; ///< timer for loop over neighbours when constructing A's for single central atom ACETimer per_atom_calc_timer; ///< timer for single compute_atom call - + ACETimer setup_timer; ACETimer forces_calc_loop_timer; ///< timer for forces calculations for single central atom ACETimer forces_calc_neighbour_timer; ///< timer for loop over neighbour atoms for force calculations - + ACETimer weights_and_theta_timer; ACETimer energy_calc_timer; ///< timer for energy calculation ACETimer total_time_calc_timer; ///< timer for total calculations of all atoms within given atomic environment system diff --git a/lib/ace/ace-evaluator/src/ace-evaluator/ace_radial.cpp b/lib/ace/ace-evaluator/src/ace-evaluator/ace_radial.cpp index d00741e..4d10912 100644 --- a/lib/ace/ace-evaluator/src/ace-evaluator/ace_radial.cpp +++ b/lib/ace/ace-evaluator/src/ace-evaluator/ace_radial.cpp @@ -400,7 +400,7 @@ DOUBLE_TYPE dsinc(DOUBLE_TYPE x) { * @param n degree * @return */ -DOUBLE_TYPE fn(DOUBLE_TYPE x, DOUBLE_TYPE rc, int n) { +DOUBLE_TYPE simplified_bessel_aux(DOUBLE_TYPE x, DOUBLE_TYPE rc, int n) { return pow(-1, n) * sqrt(2) * pi / pow(rc, 1.5) * (n + 1) * (n + 2) / sqrt(sqr(n + 1) + sqr(n + 2)) * (sinc(x * (n + 1) * pi / rc) + sinc(x * (n + 2) * pi / rc)); } @@ -412,7 +412,7 @@ DOUBLE_TYPE fn(DOUBLE_TYPE x, DOUBLE_TYPE rc, int n) { * @param n degree * @return */ -DOUBLE_TYPE dfn(DOUBLE_TYPE x, DOUBLE_TYPE rc, int n) { +DOUBLE_TYPE dsimplified_bessel_aux(DOUBLE_TYPE x, DOUBLE_TYPE rc, int n) { return pow(-1, n) * sqrt(2) * pi / pow(rc, 1.5) * (n + 1) * (n + 2) / sqrt(sqr(n + 1) + sqr(n + 2)) * (dsinc(x * (n + 1) * pi / rc) * (n + 1) * pi / rc + dsinc(x * (n + 2) * pi / rc) * (n + 2) * pi / rc); @@ -425,15 +425,15 @@ DOUBLE_TYPE dfn(DOUBLE_TYPE x, DOUBLE_TYPE rc, int n) { */ void ACERadialFunctions::simplified_bessel(DOUBLE_TYPE rc, DOUBLE_TYPE x) { if (x < rc) { - gr(0) = fn(x, rc, 0); - dgr(0) = dfn(x, rc, 0); + gr(0) = simplified_bessel_aux(x, rc, 0); + dgr(0) = dsimplified_bessel_aux(x, rc, 0); DOUBLE_TYPE d_prev = 1.0, en, dn; for (NS_TYPE n = 1; n < nradbase; n++) { en = sqr(n) * sqr(n + 2) / (4 * pow(n + 1, 4) + 1); dn = 1 - en / d_prev; - gr(n) = 1 / sqrt(dn) * (fn(x, rc, n) + sqrt(en / d_prev) * gr(n - 1)); - dgr(n) = 1 / sqrt(dn) * (dfn(x, rc, n) + sqrt(en / d_prev) * dgr(n - 1)); + gr(n) = 1 / sqrt(dn) * (simplified_bessel_aux(x, rc, n) + sqrt(en / d_prev) * gr(n - 1)); + dgr(n) = 1 / sqrt(dn) * (dsimplified_bessel_aux(x, rc, n) + sqrt(en / d_prev) * dgr(n - 1)); d_prev = dn; } } else { diff --git a/lib/ace/ace-evaluator/src/ace-evaluator/ace_radial.h b/lib/ace/ace-evaluator/src/ace-evaluator/ace_radial.h index 4b9aa46..feed27d 100644 --- a/lib/ace/ace-evaluator/src/ace-evaluator/ace_radial.h +++ b/lib/ace/ace-evaluator/src/ace-evaluator/ace_radial.h @@ -38,6 +38,11 @@ using namespace std; //typedef void (*RadialFunctions)(DOUBLE_TYPE x); typedef std::function RadialFunctions; +DOUBLE_TYPE sinc(DOUBLE_TYPE x); +DOUBLE_TYPE dsinc(DOUBLE_TYPE x); +DOUBLE_TYPE simplified_bessel_aux(DOUBLE_TYPE x, DOUBLE_TYPE rc, int n); +DOUBLE_TYPE dsimplified_bessel_aux(DOUBLE_TYPE x, DOUBLE_TYPE rc, int n); + /** * Class that implement spline interpolation and caching for radial functions */ diff --git a/lib/ace/ace-evaluator/src/ace-evaluator/ace_recursive.cpp b/lib/ace/ace-evaluator/src/ace-evaluator/ace_recursive.cpp index 77c9ba3..b515b36 100644 --- a/lib/ace/ace-evaluator/src/ace-evaluator/ace_recursive.cpp +++ b/lib/ace/ace-evaluator/src/ace-evaluator/ace_recursive.cpp @@ -854,7 +854,7 @@ ACERecursiveEvaluator::compute_atom(int i, DOUBLE_TYPE **x, const SPECIES_TYPE * const Array1D &gr = basis_set->radial_functions->gr; const Array1D &dgr = basis_set->radial_functions->dgr; - loop_over_neighbour_timer.start(); + A_construction_timer.start(); int jj_actual = 0; SPECIES_TYPE type_j = 0; @@ -973,7 +973,7 @@ ACERecursiveEvaluator::compute_atom(int i, DOUBLE_TYPE **x, const SPECIES_TYPE * } } } //now A's are constructed - loop_over_neighbour_timer.stop(); + A_construction_timer.stop(); // ==================== ENERGY ==================== @@ -1073,7 +1073,7 @@ ACERecursiveEvaluator::compute_atom(int i, DOUBLE_TYPE **x, const SPECIES_TYPE * rhos(p) += AAcur_re * dag_coefs[idx_coefs]; } - } else { + } else { // not recursive #ifdef DEBUG_ENERGY_CALCULATIONS printf("non-recursive Julia-style evaluator\n"); #endif diff --git a/lib/ace/ace-evaluator/src/ace-evaluator/ace_spherical_cart.cpp b/lib/ace/ace-evaluator/src/ace-evaluator/ace_spherical_cart.cpp index f42b3f5..aec8863 100644 --- a/lib/ace/ace-evaluator/src/ace-evaluator/ace_spherical_cart.cpp +++ b/lib/ace/ace-evaluator/src/ace-evaluator/ace_spherical_cart.cpp @@ -49,6 +49,9 @@ void ACECartesianSphericalHarmonics::init(LS_TYPE lm) { ylm.init(lmax, "ylm"); dylm.init(lmax, "dylm"); + real_ylm.init(lmax, "real_ylm"); + real_dylm.init(lmax, "real_dylm"); + pre_compute(); } @@ -216,3 +219,26 @@ void ACECartesianSphericalHarmonics::compute_ylm(DOUBLE_TYPE rx, DOUBLE_TYPE ry, } +void ACECartesianSphericalHarmonics::compute_real_ylm(DOUBLE_TYPE rx, DOUBLE_TYPE ry, DOUBLE_TYPE rz, LS_TYPE lmaxi) { + // compute complex spherical harmonics + compute_ylm(rx, ry, rz, lmaxi); + SHORT_INT_TYPE factor; + //convert ylm, dylm to real_ylm, real_dylm + for (LS_TYPE l = 0; l <= lmaxi; l++) { + // m = 0 + real_ylm(l, 0) = ylm(l, 0).real; + real_dylm(l, 0) = dylm(l, 0).real(); + + for (MS_TYPE m = 1; m <= l; m++) { + factor = m % 2 == 0 ? 1 : -1; + // m > 0 + real_ylm(l, m) = sq2 * factor * ylm(l, m).real; + real_dylm(l, m) = dylm(l, m).real() * sq2 * factor; + + // m < 0 + real_ylm(l, -m) = sq2 * factor * ylm(l, m).img; + real_dylm(l, -m) = dylm(l, m).img() * sq2 * factor; + } + } + +} \ No newline at end of file diff --git a/lib/ace/ace-evaluator/src/ace-evaluator/ace_spherical_cart.h b/lib/ace/ace-evaluator/src/ace-evaluator/ace_spherical_cart.h index 5d7c5de..98e87d1 100644 --- a/lib/ace/ace-evaluator/src/ace-evaluator/ace_spherical_cart.h +++ b/lib/ace/ace-evaluator/src/ace-evaluator/ace_spherical_cart.h @@ -44,6 +44,7 @@ const DOUBLE_TYPE sq1o4pi = 0.28209479177387814347; // sqrt(1/(4*pi)) const DOUBLE_TYPE sq4pi = 3.54490770181103176384; // sqrt(4*pi) const DOUBLE_TYPE sq3 = 1.73205080756887719318;//sqrt(3), numpy const DOUBLE_TYPE sq3o2 = 1.22474487139158894067;//sqrt(3/2), numpy +const DOUBLE_TYPE sq2 = 1.4142135623730951;//sqrt(2), numpy //definition of common factor for spherical harmonics = Y00 //const DOUBLE_TYPE Y00 = sq1o4pi; @@ -128,6 +129,9 @@ class ACECartesianSphericalHarmonics { Array2DLM ylm; ///< Values of all spherical harmonics after \code compute_ylm(rx,ry,rz, lmaxi) \endcode call Array2DLM dylm;///< Values of gradients of all spherical harmonics after \code compute_ylm(rx,ry,rz, lmaxi) \endcode call + Array2DLM real_ylm; ///< Values of all real spherical harmonics after \code compute_ylm(rx,ry,rz, lmaxi) \endcode call + Array2DLM real_dylm;///< Values of gradients of all real spherical harmonics after \code compute_ylm(rx,ry,rz, lmaxi) \endcode call + void compute_real_ylm(DOUBLE_TYPE rx, DOUBLE_TYPE ry, DOUBLE_TYPE rz, LS_TYPE lmaxi); }; diff --git a/lib/ace/ace-evaluator/src/ace-evaluator/ace_version.h b/lib/ace/ace-evaluator/src/ace-evaluator/ace_version.h index 8e8ffc7..a3f30e4 100644 --- a/lib/ace/ace-evaluator/src/ace-evaluator/ace_version.h +++ b/lib/ace/ace-evaluator/src/ace-evaluator/ace_version.h @@ -31,9 +31,9 @@ #ifndef ACE_VERSION_H #define ACE_VERSION_H -#define VERSION_YEAR 2023 -#define VERSION_MONTH 11 -#define VERSION_DAY 25 +#define VERSION_YEAR 2024 +#define VERSION_MONTH 7 +#define VERSION_DAY 4 #endif //ACE_VERSION_Hls diff --git a/lib/ace/ace-evaluator/src/extra/ace_calculator.cpp b/lib/ace/ace-evaluator/src/extra/ace_calculator.cpp index d3eeb9b..dcac1de 100644 --- a/lib/ace/ace-evaluator/src/extra/ace_calculator.cpp +++ b/lib/ace/ace-evaluator/src/extra/ace_calculator.cpp @@ -206,25 +206,30 @@ void ACECalculator::compute(ACEAtomicEnvironment &atomic_environment, bool comp #ifdef FINE_TIMING if (verbose) { - printf(" Total time: %ld microseconds\n", evaluator->total_time_calc_timer.as_microseconds()); - printf("Per atom time: %ld microseconds\n", + printf("(Calculator)Total time: %ld microseconds\n", evaluator->total_time_calc_timer.as_microseconds()); + printf("(Evaluator) Total time/at: %ld microseconds\n", evaluator->per_atom_calc_timer.as_microseconds() / atomic_environment.n_atoms_real); + printf("setup/atom: %ld microseconds\n", + evaluator->setup_timer.as_microseconds() / atomic_environment.n_atoms_real); - printf("Loop_over_nei/atom: %ld microseconds\n", - evaluator->loop_over_neighbour_timer.as_microseconds() / atomic_environment.n_atoms_real); + printf("A_construction/atom: %ld microseconds\n", + evaluator->A_construction_timer.as_microseconds() / atomic_environment.n_atoms_real); printf(" Energy/atom: %ld microseconds\n", evaluator->energy_calc_timer.as_microseconds() / atomic_environment.n_atoms_real); + printf("Weights and theta/atom: %ld microseconds\n", + evaluator->weights_and_theta_timer.as_microseconds() / atomic_environment.n_atoms_real); + printf(" Forces/atom: %ld microseconds\n", evaluator->forces_calc_loop_timer.as_microseconds() / atomic_environment.n_atoms_real); // printf("phi_recalcs/atom: %ld microseconds\n", // evaluator->phi_recalc_timer.as_microseconds() / atomic_environment.n_atoms_real); - printf(" forces_neig: %ld microseconds\n", - evaluator->forces_calc_neighbour_timer.as_microseconds() / atomic_environment.n_atoms_real); +// printf(" forces_neig: %ld microseconds\n", +// evaluator->forces_calc_neighbour_timer.as_microseconds() / atomic_environment.n_atoms_real); } #endif diff --git a/lib/ace/src/ace/ace_b_evaluator.cpp b/lib/ace/src/ace/ace_b_evaluator.cpp index 993d640..e09fdd8 100644 --- a/lib/ace/src/ace/ace_b_evaluator.cpp +++ b/lib/ace/src/ace/ace_b_evaluator.cpp @@ -106,6 +106,7 @@ ACEBEvaluator::compute_atom(int i, DOUBLE_TYPE **x, const SPECIES_TYPE *type, co throw std::invalid_argument("ACEBEvaluator: basis set is not assigned"); } per_atom_calc_timer.start(); + setup_timer.start(); DOUBLE_TYPE evdwl = 0, evdwl_cut = 0, rho_core = 0; DOUBLE_TYPE r_norm; DOUBLE_TYPE xn, yn, zn, r_xyz; @@ -213,8 +214,7 @@ ACEBEvaluator::compute_atom(int i, DOUBLE_TYPE **x, const SPECIES_TYPE *type, co const Array1D &gr = basis_set->radial_functions->gr; const Array1D &dgr = basis_set->radial_functions->dgr; - - loop_over_neighbour_timer.start(); + setup_timer.stop(); int jj_actual = 0; SPECIES_TYPE type_j = 0; @@ -267,6 +267,8 @@ ACEBEvaluator::compute_atom(int i, DOUBLE_TYPE **x, const SPECIES_TYPE *type, co //ALGORITHM 1: Atomic base A + A_construction_timer.start(); + for (jj = 0; jj < jnum_actual; ++jj) { r_norm = r_norms(jj); mu_j = elements(jj); @@ -335,7 +337,7 @@ ACEBEvaluator::compute_atom(int i, DOUBLE_TYPE **x, const SPECIES_TYPE *type, co } } } //now A's are constructed - loop_over_neighbour_timer.stop(); + A_construction_timer.stop(); // ==================== ENERGY ==================== diff --git a/lib/ace/src/ace/grace_fs_evaluator.cpp b/lib/ace/src/ace/grace_fs_evaluator.cpp new file mode 100644 index 0000000..a970051 --- /dev/null +++ b/lib/ace/src/ace/grace_fs_evaluator.cpp @@ -0,0 +1,870 @@ +// +// Created by Yury Lysogorskiy on 24.06.24 +// +#include "ace/grace_fs_evaluator.h" +#include "ace-evaluator/ace_types.h" +#include "ace/ace_yaml_input.h" +#include "ace-evaluator/ace_radial.h" +#include "cnpy/cnpy.h" + +#define sqr(x) ((x)*(x)) +const DOUBLE_TYPE pi = 3.14159265358979323846264338327950288419; // pi + +void GRACEFSEmbeddingSpecification::from_YAML(YAML_PACE::Node emb_yaml) { + this->type = emb_yaml["type"].as(); + this->FS_parameters = emb_yaml["params"].as>(); + if (this->FS_parameters.size() % 2 == 0) + this->ndensity = this->FS_parameters.size() / 2; + else { + throw invalid_argument("Number of parameters in emb_spec::params is not even"); + } +// this->ndensity = emb_yaml["ndensity"].as(); +} + +void GRACEFSRadialFunction::from_YAML(YAML_PACE::Node bond_yaml) { + + this->radbasename = bond_yaml["radbasename"].as(); + + this->nradmax = bond_yaml["nradmax"].as(); + this->nradbasemax = bond_yaml["n_rad_base"].as(); + + this->crad_shape = bond_yaml["crad_shape"].as>(); // [n_radial][lmax+1][n_rad_base] + this->Z_shape = bond_yaml["Z_shape"].as>(); // [n_elements][n_radial] + this->nelemets = this->Z_shape.at(0); + + this->rad_lamba = 0; // TODO: check ? + this->lmax = crad_shape[1] - 1; // crad = [n,l,k], crad_shape[..., lmax+1, ...] + + auto crad_flat = bond_yaml["crad"].as>(); + this->crad.init(crad_shape[0], crad_shape[1], crad_shape[2]); + this->crad.set_flatten_vector(crad_flat); + + auto Z_flat = bond_yaml["Z"].as>(); + this->Z.init(Z_shape[0], Z_shape[1], "Z"); + Z.set_flatten_vector(Z_flat); + + + rcut = bond_yaml["cutoff"].as(); + if (bond_yaml["dcut"]) dcut = bond_yaml["dcut"].as(); + + if (bond_yaml["rcut_in"]) rcut_in = bond_yaml["rcut_in"].as(); + if (bond_yaml["dcut_in"]) dcut_in = bond_yaml["dcut_in"].as(); +} + +void GRACEFSBasisFunction::from_YAML(YAML_PACE::Node node) { + this->mu0 = node["mu0"].as(); + + this->ns = node["ns"].as(); + this->ls = node["ls"].as>(); +// if (this->ns.size() != this->ls.size()) +// throw invalid_argument("ns.size!=ls.size"); + this->rank = this->ls.get_size(); + + + this->ms_combs = node["ms_combs"].as>(); + this->gen_cgs = node["gen_cgs"].as>(); + if (this->gen_cgs.get_size() * this->rank != this->ms_combs.get_size()) + throw invalid_argument("gen_cg.size * rank != ms_combs.size"); + this->num_ms_combs = this->gen_cgs.get_size(); + + this->ndensity = node["ndensity"].as(); + this->coeff = node["coeff"].as>(); + if (this->ndensity != this->coeff.get_size()) + throw invalid_argument("ndensity != coeff.size"); +} + +void GRACEFSBasisFunction::print() const { + cout << "TDACEBasisFunction(mu0=" << this->mu0 << ", rank=" << (int) this->rank << endl; + int rank = this->rank; + + for (int ms_ind = 0; ms_ind < this->num_ms_combs; ms_ind++) { + for (int r = 0; r < rank; r++) + cout << "A(" << this->ns << "," << this->ls(r) << "," << this->ms_combs(ms_ind * rank + r) << ")"; + cout << " * " << this->gen_cgs(ms_ind) << endl; + } + cout << ")" << endl; +} + +GRACEFSBasisSet::GRACEFSBasisSet(const string &filename) { + this->load(filename); +} + +void GRACEFSBasisSet::load(const string &filename) { + this->filename = filename; + + if (!if_file_exist(filename)) { + stringstream s; + s << "Potential file " << filename << " doesn't exists"; + cerr << "Exception: " << s.str(); + throw invalid_argument(s.str()); + } + + //load the file with yaml + YAML_PACE::Node YAML_input = YAML_PACE::LoadFile(filename); + this->elements_name = YAML_input["elements"].as>(); + this->nelements = this->elements_name.size(); + for (int i = 0; i < this->nelements; i++) + this->elements_to_index_map[this->elements_name[i]] = i; + + this->embedding_specifications.from_YAML(YAML_input["emb_spec"]); + + this->radial_functions.from_YAML(YAML_input["radial_basis"]); + + if (YAML_input["nnorm"]) + this->nnorm = YAML_input["nnorm"].as(); //inv_avg_n_neigh + + if (this->radial_functions.nelemets != this->nelements) + throw invalid_argument("radial_basis.Z_shape != elements.size"); + + // load functions + this->basis.resize(this->nelements); + RANK_TYPE rankmax = 0; + for (const auto &mu_functions_node: YAML_input["functions"]) { + SPECIES_TYPE mu0 = mu_functions_node.first.as(); + const auto &functions_list = mu_functions_node.second; + vector basis_functions_vec; + for (const auto &func_node: functions_list) { + GRACEFSBasisFunction func; + func.from_YAML(func_node); + basis_functions_vec.emplace_back(func); + if (func.rank > rankmax) + rankmax = func.rank; + } + this->basis.at(mu0) = basis_functions_vec; + } + + //////////////// initialize variables ///////////////// + this->nradmax = this->radial_functions.nradmax; + this->lmax = this->radial_functions.lmax; + this->nradbase = this->radial_functions.nradbasemax; + this->ndensitymax = this->embedding_specifications.ndensity; + this->cutoffmax = this->radial_functions.rcut; + this->rankmax = rankmax; + + // compute max_dB_array_size (among different central species) + this->max_dB_array_size = 0; + int cur_ms_rank_size = 0; + + for (SPECIES_TYPE mu = 0; mu < nelements; ++mu) { + cur_ms_rank_size = 0; + + for (const auto &func: this->basis.at(mu)) + cur_ms_rank_size += func.rank * func.num_ms_combs; + + if (cur_ms_rank_size > max_dB_array_size) + max_dB_array_size = cur_ms_rank_size; + } + + this->radial_functions.init(); + this->spherical_harmonics.init(this->lmax); + + this->convert_to_flatten_arrays(); + + //load E0, shift and scale + if (YAML_input["E0"]) this->E0_shift = YAML_input["E0"].as>(); + else this->E0_shift.resize(nelements, 0.0); + + if (YAML_input["shift"]) this->shift = YAML_input["shift"].as(); + else this->shift = 0.0; + + if (YAML_input["scale"]) this->scale = YAML_input["scale"].as(); + else this->scale = 1.0; +} + +void GRACEFSBasisSet::convert_to_flatten_arrays() { + ranks_flatten.resize(nelements); + ndensity_flatten.resize(nelements); + num_ms_combs_flatten.resize(nelements); + ns_flatten.resize(nelements); + ls_flatten.resize(nelements); + ms_combs_flatten.resize(nelements); + gen_cgs_flatten.resize(nelements); + coeff_flatten.resize(nelements); + + func_ind_to_ls.resize(nelements); + func_ind_to_ms_combs.resize(nelements); + func_ind_to_gen_cgs.resize(nelements); + func_ind_to_coeff.resize(nelements); + + for (int el = 0; el < nelements; el++) { +// printf("Element: %d\n", el); + + const auto num_funcs = basis[el].size(); + ranks_flatten[el].init(num_funcs, "ranks_flatten"); + ndensity_flatten[el].init(num_funcs, "ndensity_flatten"); + num_ms_combs_flatten[el].init(num_funcs, "num_ms_combs_flatten"); + + ns_flatten[el].init(num_funcs, "ns_flatten"); + + int num_ls = 0; + int num_ms = 0; + int num_gen_cgs = 0; + int num_coeff = 0; + for (const auto &func: basis[el]) { + num_ls += func.rank; + num_ms += func.num_ms_combs * func.rank; + num_gen_cgs += func.num_ms_combs; + num_coeff += func.ndensity; + } +// printf("num_ls=%d\n", num_ls); +// printf("num_ms=%d\n", num_ms); +// printf("num_gen_cgs=%d\n", num_gen_cgs); +// printf("num_coeff=%d\n", num_coeff); + + ls_flatten[el].init(num_ls, "ls_flatten"); + ms_combs_flatten[el].init(num_ms, "ms_combs_flatten"); + + gen_cgs_flatten[el].init(num_gen_cgs, "gen_cgs_flatten"); + coeff_flatten[el].init(num_coeff, "coeff_flatten"); + + func_ind_to_ls[el].init(num_funcs, "func_ind_to_ls"); + func_ind_to_ms_combs[el].init(num_funcs, "func_ind_to_ms_combs"); + func_ind_to_gen_cgs[el].init(num_funcs, "func_ind_to_gen_cgs"); + func_ind_to_coeff[el].init(num_funcs, "func_ind_to_coeff"); + } + + // population of arrays + for (int el = 0; el < nelements; el++) { + int ind_ls = 0; + int ind_ms = 0; + int ind_gen_cgs = 0; + int ind_coeff = 0; + + int func_ind = 0; + for (const auto &func: basis[el]) { + ranks_flatten[el](func_ind) = func.rank; + ndensity_flatten[el](func_ind) = func.ndensity; + num_ms_combs_flatten[el](func_ind) = func.num_ms_combs; + ns_flatten[el](func_ind) = func.ns; + + // ls_flatten + func_ind_to_ls[el](func_ind) = ind_ls; + ls_flatten[el].move_from(func.ls, ind_ls); + ind_ls += func.rank; + + // ms_combs_flatten + func_ind_to_ms_combs[el](func_ind) = ind_ms; + ms_combs_flatten[el].move_from(func.ms_combs, ind_ms); + ind_ms += func.num_ms_combs * func.rank; + + // gen_cgs_flatten + func_ind_to_gen_cgs[el](func_ind) = ind_gen_cgs; + gen_cgs_flatten[el].move_from(func.gen_cgs, ind_gen_cgs); + ind_gen_cgs += func.num_ms_combs; + + // coeff_flatten + func_ind_to_coeff[el](func_ind) = ind_coeff; + coeff_flatten[el].move_from(func.coeff, ind_coeff); + ind_coeff += func.ndensity; + + func_ind++; + } + } +} + +void GRACEFSBasisSet::FS_values_and_derivatives(Array1D &rhos, DOUBLE_TYPE &value, + Array1D &derivatives, + SPECIES_TYPE mu_i) { + DOUBLE_TYPE F, DF = 0, wpre, mexp; + DENSITY_TYPE ndensity = this->ndensitymax; + + for (int p = 0; p < ndensity; p++) { + wpre = this->embedding_specifications.FS_parameters[p * 2 + 0]; + mexp = this->embedding_specifications.FS_parameters[p * 2 + 1]; + string npoti = this->embedding_specifications.type; + + if (npoti == "FinnisSinclair") + Fexp(rhos(p), mexp, F, DF); + else if (npoti == "FinnisSinclairShiftedScaled") + FexpShiftedScaled(rhos(p), mexp, F, DF); + + value += F * wpre; // * weight (wpre) + derivatives(p) = DF * wpre;// * weight (wpre) + } +} + +void GRACEFSBasisSet::print_functions() { + for (int el = 0; el < nelements; el++) { + printf("============================= Element: %d =====================\n", el); + for (const auto &func: basis[el]) { + func.print(); + } + } +} + +void GRACEFSBEvaluator::set_basis(GRACEFSBasisSet &basis_set) { + this->basis_set = basis_set; + init(this->basis_set); +} + +void GRACEFSBEvaluator::init(GRACEFSBasisSet &basis_set) { + A.init(basis_set.nradmax + 1, basis_set.lmax + 1, "A"); + + rhos.init(basis_set.ndensitymax, "rhos"); + dF_drho.init(basis_set.ndensitymax, "dF_drho"); + + weights.init(basis_set.nradmax + 1, basis_set.lmax + 1, "weights"); + + + DG_cache.init(1, basis_set.nradbase, "DG_cache"); + DG_cache.fill(0); + + R_cache.init(1, basis_set.nradmax, basis_set.lmax + 1, "R_cache"); + R_cache.fill(0); + + DR_cache.init(1, basis_set.nradmax, basis_set.lmax + 1, "DR_cache"); + DR_cache.fill(0); + + Y_cache.init(1, basis_set.lmax + 1, "Y_cache"); + Y_cache.fill(0); + + DY_cache.init(1, basis_set.lmax + 1, "dY_dense_cache"); + DY_cache.fill({0., 0, 0}); + + //hard-core repulsion + DCR_cache.init(1, "DCR_cache"); + DCR_cache.fill(0); + + dB_flatten.init(basis_set.max_dB_array_size, "dB_flatten"); +} + + +void GRACEFSBEvaluator::resize_neighbours_cache(int max_jnum) { + if (R_cache.get_dim(0) < max_jnum) { + + //TODO: implement grow + R_cache.resize(max_jnum, basis_set.nradmax, basis_set.lmax + 1); + R_cache.fill(0); + + DR_cache.resize(max_jnum, basis_set.nradmax, basis_set.lmax + 1); + DR_cache.fill(0); + + DG_cache.resize(max_jnum, basis_set.nradbase); + DG_cache.fill(0); + + Y_cache.resize(max_jnum, basis_set.lmax + 1); + Y_cache.fill(0); + + DY_cache.resize(max_jnum, basis_set.lmax + 1); + DY_cache.fill({0, 0, 0}); + + //hard-core repulsion + DCR_cache.resize(max_jnum); + DCR_cache.fill(0); + + r_norms.resize(max_jnum); + inv_r_norms.resize(max_jnum); + rhats.resize(max_jnum, 3); + elements.resize(max_jnum); + } +} + + +void GRACEFSBEvaluator::compute_atom(int i, DOUBLE_TYPE **x, const SPECIES_TYPE *type, const int jnum, const int *jlist) { + per_atom_calc_timer.start(); + + setup_timer.start(); + DOUBLE_TYPE evdwl = 0; + DOUBLE_TYPE r_norm; + DOUBLE_TYPE xn, yn, zn, r_xyz; + DOUBLE_TYPE R, R_over_r, DR; + DOUBLE_TYPE *r_hat; + + SPECIES_TYPE mu_j; + RANK_TYPE r, rank, t; + NS_TYPE n; + LS_TYPE l; + MS_TYPE m, m_t; + + NS_TYPE ns; + LS_TYPE *ls; + MS_TYPE *ms; + + int j, jj, func_ind, ms_ind; + + DOUBLE_TYPE Y, Y_DR; + DOUBLE_TYPE B; + DOUBLE_TYPE dB; + Array1D A_cache(basis_set.rankmax); + ACEDRealYcomponent grad_phi_nlm, DY; + + //size is +1 of max to avoid out-of-boundary array access in double-triangular scheme + Array1D A_forward_prod(basis_set.rankmax + 1); + Array1D A_backward_prod(basis_set.rankmax + 1); + + DOUBLE_TYPE inv_r_norm; + const DOUBLE_TYPE xtmp = x[i][0]; + const DOUBLE_TYPE ytmp = x[i][1]; + const DOUBLE_TYPE ztmp = x[i][2]; + DOUBLE_TYPE f_ji[3]; + + bool is_element_mapping = element_type_mapping.get_size() > 0; + SPECIES_TYPE mu_i; + if (is_element_mapping) + mu_i = element_type_mapping(type[i]); + else + mu_i = type[i]; + + const SHORT_INT_TYPE total_basis_size = basis_set.basis[mu_i].size(); + + const DENSITY_TYPE ndensity = basis_set.embedding_specifications.ndensity; + + neighbours_forces.resize(jnum, 3); + neighbours_forces.fill(0.0); + + weights.fill(0); + A.fill(0); + rhos.fill(0); + dF_drho.fill(0); + +#ifdef EXTRA_C_PROJECTIONS + if (this->compute_projections) { + projections.init(total_basis_size, "projections"); + projections.fill(0); + } +#endif + + setup_timer.stop(); + + A_construction_timer.start(); + const auto &ylm = basis_set.spherical_harmonics.real_ylm; + const auto &dylm = basis_set.spherical_harmonics.real_dylm; + + const auto &fr = basis_set.radial_functions.fr; + const auto &dfr = basis_set.radial_functions.dfr; + + const auto &ranks_flatten = basis_set.ranks_flatten[mu_i]; + const auto &ns_flatten = basis_set.ns_flatten[mu_i]; + auto &ls_flatten = basis_set.ls_flatten[mu_i]; + const auto &func_ind_to_ls = basis_set.func_ind_to_ls[mu_i]; + const auto &num_ms_combs_flatten = basis_set.num_ms_combs_flatten[mu_i]; + const auto &func_ind_to_ms_combs = basis_set.func_ind_to_ms_combs[mu_i]; + auto &ms_combs_flatten = basis_set.ms_combs_flatten[mu_i]; + const auto &func_ind_to_gen_cgs = basis_set.func_ind_to_gen_cgs[mu_i]; + const auto &func_ind_to_coeff = basis_set.func_ind_to_coeff[mu_i]; + const auto &gen_cgs_flatten = basis_set.gen_cgs_flatten[mu_i]; + const auto &coeff_flatten = basis_set.coeff_flatten[mu_i]; + int jj_actual = 0; + SPECIES_TYPE type_j = 0; + Array1D neighbour_index_mapping(jnum); // jj_actual -> jj + //loop over neighbours, compute distance, consider only atoms within with r= basis_set.cutoffmax) + continue; + + inv_r_norm = 1 / r_xyz; + + r_norms(jj_actual) = r_xyz; + inv_r_norms(jj_actual) = inv_r_norm; + rhats(jj_actual, 0) = xn * inv_r_norm; + rhats(jj_actual, 1) = yn * inv_r_norm; + rhats(jj_actual, 2) = zn * inv_r_norm; + elements(jj_actual) = mu_j; + neighbour_index_mapping(jj_actual) = jj; + jj_actual++; + } + + int jnum_actual = jj_actual; + + //ALGORITHM 1: Atomic base A + for (jj = 0; jj < jnum_actual; ++jj) { + r_norm = r_norms(jj); + mu_j = elements(jj); + r_hat = &rhats(jj, 0); + + //proxies + auto &Y_jj = Y_cache(jj); + auto &DY_jj = DY_cache(jj); + + basis_set.radial_functions.evaluate(r_norm, mu_i, mu_j); + basis_set.spherical_harmonics.compute_real_ylm(r_hat[0], r_hat[1], r_hat[2], basis_set.lmax); + + //loop for computing A's + // for rank > 1 + // A_construction_timer2.start(); + + for (n = 0; n < basis_set.nradmax; n++) { + auto &A_lm = A(n); + for (l = 0; l <= basis_set.lmax; l++) { + R = fr(n, l) * basis_set.nnorm; + + DR_cache(jj, n, l) = dfr(n, l) * basis_set.nnorm; + R_cache(jj, n, l) = R; + + for (m = -l; m <= l; m++) { + Y = ylm(l, m); + A_lm(l, m) += R * Y; //accumulation sum over neighbours + Y_jj(l, m) = Y; + DY_jj(l, m) = dylm(l, m); + } + + } + } +// A_construction_timer2.stop(); + + } //end loop over neighbours + A_construction_timer.stop(); + + //now A's are constructed (no need for complex conjugation form -m, since it was constructed + + // ==================== ENERGY ==================== + + energy_calc_timer.start(); + //ALGORITHM 2: Basis functions B with iterative product and density rho(p) calculation + // all rank>=1 + int func_ms_ind = 0; + int func_ms_t_ind = 0;// index for dB + int num_ms_combs = 0; + + + for (func_ind = 0; func_ind < total_basis_size; ++func_ind) { + rank = ranks_flatten(func_ind); + r = rank - 1; + ns = ns_flatten(func_ind); + ls = &ls_flatten(func_ind_to_ls(func_ind)); + + //loop over {ms} combinations in sum + num_ms_combs = num_ms_combs_flatten(func_ind); + const auto ms_ind_shift = func_ind_to_ms_combs(func_ind); + const auto gen_cgs_shift = func_ind_to_gen_cgs(func_ind); + const auto coeff_shift = func_ind_to_coeff(func_ind); + + for (ms_ind = 0; ms_ind < num_ms_combs; ++ms_ind, ++func_ms_ind) { + ms = &ms_combs_flatten(ms_ind_shift + ms_ind * rank); + + //loop over m, collect B = product of A with given ms + A_forward_prod(0) = 1; + A_backward_prod(r) = 1; + + //fill forward A-product triangle + for (t = 0; t < rank; t++) { + A_cache(t) = A(ns - 1, ls[t], ms[t]); + A_forward_prod(t + 1) = A_forward_prod(t) * A_cache(t); + } + + B = A_forward_prod(t); + + //fill backward A-product triangle + for (t = r; t >= 1; t--) + A_backward_prod(t - 1) = A_backward_prod(t) * A_cache(t); + + for (t = 0; t < rank; ++t, ++func_ms_t_ind) { + dB = A_forward_prod(t) * A_backward_prod(t); //dB - product of all A's except t-th + dB_flatten(func_ms_t_ind) = dB; + } + +#ifdef EXTRA_C_PROJECTIONS + if (this->compute_projections) { + //aggregate C-projections separately + projections(func_ind) += B * gen_cgs_flatten(gen_cgs_shift + ms_ind); + } +#endif + + for (DENSITY_TYPE p = 0; p < ndensity; ++p) { + //real-part only multiplication + rhos(p) += B * gen_cgs_flatten(gen_cgs_shift + ms_ind) * + coeff_flatten(coeff_shift + p); + } + }//end of loop over {ms} combinations in sum + } // end loop for rank>=1 + + basis_set.FS_values_and_derivatives(rhos, evdwl, dF_drho, mu_i); + energy_calc_timer.stop(); + + //ALGORITHM 3: Weights and theta calculation + // all ranks>=1 + weights_and_theta_timer.start(); + func_ms_ind = 0; + func_ms_t_ind = 0;// index for dB + DOUBLE_TYPE theta; + for (func_ind = 0; func_ind < total_basis_size; ++func_ind) { + rank = ranks_flatten(func_ind); + ns = ns_flatten(func_ind); + ls = &ls_flatten(func_ind_to_ls(func_ind)); + num_ms_combs = num_ms_combs_flatten(func_ind); + const auto ms_ind_shift = func_ind_to_ms_combs(func_ind); + const auto gen_cgs_shift = func_ind_to_gen_cgs(func_ind); + const auto coeff_shift = func_ind_to_coeff(func_ind); + + for (ms_ind = 0; ms_ind < num_ms_combs; ++ms_ind, ++func_ms_ind) { +// ms = &func.ms_combs.get_data()[ms_ind * rank]; // current ms-combination (of length = rank) + ms = &ms_combs_flatten(ms_ind_shift + ms_ind * rank); + + theta = 0; + for (DENSITY_TYPE p = 0; p < ndensity; ++p) + theta += dF_drho(p) * gen_cgs_flatten(gen_cgs_shift + ms_ind) * + coeff_flatten(coeff_shift + p); + + + for (t = 0; t < rank; ++t, ++func_ms_t_ind) { + m_t = ms[t]; + dB = dB_flatten(func_ms_t_ind); + weights(ns - 1, ls[t], m_t) += theta * dB; //Theta_array(func_ms_ind); + } + } + } + weights_and_theta_timer.stop(); + + // ==================== FORCES ==================== + // loop over neighbour atoms for force calculations + forces_calc_loop_timer.start(); + for (jj = 0; jj < jnum_actual; ++jj) { + r_hat = &rhats(jj, 0); + inv_r_norm = inv_r_norms(jj); + const auto &Y_cache_jj = Y_cache(jj); + const auto &DY_cache_jj = DY_cache(jj); + + f_ji[0] = f_ji[1] = f_ji[2] = 0; + + //for rank >= 1 + for (n = 0; n < basis_set.nradmax; n++) { + for (l = 0; l <= basis_set.lmax; l++) { + R_over_r = R_cache(jj, n, l) * inv_r_norm; + DR = DR_cache(jj, n, l); + + // for all m + for (m = -l; m <= l; m++) { + auto w = weights(n, l, m); + if (w == 0) + continue; + DY = DY_cache_jj(l, m); + Y_DR = Y_cache_jj(l, m) * DR; + + grad_phi_nlm.a[0] = Y_DR * r_hat[0] + DY.a[0] * R_over_r; + grad_phi_nlm.a[1] = Y_DR * r_hat[1] + DY.a[1] * R_over_r; + grad_phi_nlm.a[2] = Y_DR * r_hat[2] + DY.a[2] * R_over_r; + + f_ji[0] += w * grad_phi_nlm.a[0]; + f_ji[1] += w * grad_phi_nlm.a[1]; + f_ji[2] += w * grad_phi_nlm.a[2]; + } + } + } + neighbours_forces(neighbour_index_mapping(jj), 0) = basis_set.scale * f_ji[0]; + neighbours_forces(neighbour_index_mapping(jj), 1) = basis_set.scale * f_ji[1]; + neighbours_forces(neighbour_index_mapping(jj), 2) = basis_set.scale * f_ji[2]; + + } // end for-loop over neighbour atoms for force calculations + forces_calc_loop_timer.stop(); + + e_atom = basis_set.scale * evdwl + basis_set.shift + basis_set.E0_shift.at(mu_i); + +#ifdef EXTRA_C_PROJECTIONS + if (this->compute_projections) { + //check if active set is loaded + // use dE_dc or projections as asi_vector + if (A_active_set_inv.find(mu_i) != A_active_set_inv.end()) { + Array1D &asi_vector = this->projections; + // get inverted active set for current species type + const auto &A_as_inv = A_active_set_inv.at(mu_i); + + DOUBLE_TYPE gamma_max = 0; + for (int i = 0; i < A_as_inv.get_dim(0); i++) { + DOUBLE_TYPE current_gamma = 0; + // compute row-matrix-multiplication asi_vector * A_as_inv (transposed matrix) + for (int k = 0; k < asi_vector.get_dim(0); k++) + current_gamma += asi_vector(k) * A_as_inv(i, k); + + if (abs(current_gamma) > gamma_max) + gamma_max = abs(current_gamma); + } + + max_gamma_grade = gamma_max; + } + } +#endif + per_atom_calc_timer.stop(); +} + +void GRACEFSBEvaluator::init_timers() { + A_construction_timer.init(); + A_construction_timer2.init(); + forces_calc_loop_timer.init(); + energy_calc_timer.init(); + per_atom_calc_timer.init(); + total_time_calc_timer.init(); + weights_and_theta_timer.init(); + setup_timer.init(); +} + + +void GRACEFSBEvaluator::load_active_set(const string &asi_filename) { + cnpy::npz_t asi_npz = cnpy::npz_load(asi_filename); + if (asi_npz.size() != this->basis_set.nelements) { + stringstream ss; + ss << "Number of species types in ASI `" << asi_filename << "` (" << asi_npz.size() << ")"; + ss << "not equal to number of species in TDACEBBassiSet (" << this->basis_set.nelements << ")"; + throw std::runtime_error(ss.str()); + } + + for (auto &kv: asi_npz) { + auto element_name = kv.first; + SPECIES_TYPE st = this->basis_set.elements_to_index_map.at(element_name); + auto shape = kv.second.shape; + // auto_determine extrapolation grade type: linear or non-linear +// validate_ASI_square_shape(st, shape); + if (shape.at(0) != shape.at(1)) { + stringstream ss; + ss << "Active Set Inverted for element `" << element_name << "`:"; + ss << "should be square matrix, but has shape (" << shape.at(0) << ", " << shape.at(1) << ")"; + throw runtime_error(ss.str()); + } +// validate_ASI_shape(element_name, st, shape); + int expected_ASI_size = this->basis_set.basis[st].size(); + if (expected_ASI_size != shape.at(0)) { + stringstream ss; + ss << "Active Set Inverted for element `" << element_name << "`:"; + ss << "expected shape: (" << expected_ASI_size << ", " << expected_ASI_size << ") , but has shape (" + << shape.at(0) << ", " << shape.at(1) << ")"; + throw runtime_error(ss.str()); + } + + Array2D A0_inv(shape.at(0), shape.at(1), element_name); + auto data_vec = kv.second.as_vec(); + A0_inv.set_flatten_vector(data_vec); + //transpose matrix to speed-up vec-mat multiplication + Array2D A0_inv_transpose(A0_inv.get_dim(1), A0_inv.get_dim(0)); + + for (int i = 0; i < A0_inv.get_dim(0); i++) + for (int j = 0; j < A0_inv.get_dim(1); j++) + A0_inv_transpose(j, i) = A0_inv(i, j); + + this->A_active_set_inv[st] = A0_inv_transpose; + } + +// resize_projections();// no need, all projections are the same length +} + + +void GRACEFSRadialFunction::init() { + + gr.init(nradbasemax, "gr"); + dgr.init(nradbasemax, "dgr"); + + fr.init(nradmax, lmax + 1, "fr"); + dfr.init(nradmax, lmax + 1, "dfr"); + +// splines_gk.init(nelements, nelements, "splines_gk"); +// splines_rnl.init(nelements, nelements, "splines_rnl"); + setuplookupRadspline(); +} + + +/** + * Simplified Bessel function + * @param rc + * @param x + */ +void GRACEFSRadialFunction::simplified_bessel(DOUBLE_TYPE rc, DOUBLE_TYPE x) { + if (x < rc) { + gr(0) = simplified_bessel_aux(x, rc, 0); + dgr(0) = dsimplified_bessel_aux(x, rc, 0); + + DOUBLE_TYPE d_prev = 1.0, en, dn; + for (NS_TYPE n = 1; n < this->nradbasemax; n++) { + en = sqr(n) * sqr(n + 2) / (4 * pow(n + 1, 4) + 1); + dn = 1 - en / d_prev; + gr(n) = 1 / sqrt(dn) * (simplified_bessel_aux(x, rc, n) + sqrt(en / d_prev) * gr(n - 1)); + dgr(n) = 1 / sqrt(dn) * (dsimplified_bessel_aux(x, rc, n) + sqrt(en / d_prev) * dgr(n - 1)); + d_prev = dn; + } + } else { + gr.fill(0); + dgr.fill(0); + } +} + +void GRACEFSRadialFunction::radbase(DOUBLE_TYPE lam, DOUBLE_TYPE cut, DOUBLE_TYPE dcut, string radbasename, DOUBLE_TYPE r, + DOUBLE_TYPE cut_in, DOUBLE_TYPE dcut_in) { + /*lam is given by the formula (24), that contains cut */ + if (r <= cut_in - dcut_in || r >= cut) { + gr.fill(0); + dgr.fill(0); + } else { // cut_in < r < cut + if (radbasename == "SBessel") { + simplified_bessel(cut, r); + } else { + throw invalid_argument("Unknown radial basis function name: " + radbasename); + } + } +} + +void GRACEFSRadialFunction::all_radfunc(DOUBLE_TYPE r) { + + // set up radial functions + radbase(rad_lamba, rcut, dcut, radbasename, r, 0, 0); //update gr, dgr + radfunc(); // update fr(nr, l), dfr(nr, l) +} + +void GRACEFSRadialFunction::radfunc() { + DOUBLE_TYPE frval, dfrval; + for (NS_TYPE n = 0; n < nradmax; n++) { + for (LS_TYPE l = 0; l <= lmax; l++) { + frval = 0.0; + dfrval = 0.0; + for (NS_TYPE k = 0; k < nradbasemax; k++) { + frval += crad(n, l, k) * gr(k); + dfrval += crad(n, l, k) * dgr(k); + } + // IMPORTANT!!! MULTIPLICATION BY Z(mu_j,n) will happen in evaluate, + // otherwise we should store too much almost identical SplineInterpolators + fr(n, l) = frval; + dfr(n, l) = dfrval; + } + } +} + +void GRACEFSRadialFunction::setuplookupRadspline() { + using namespace std::placeholders; + splines_gk.setupSplines(gr.get_size(), + std::bind(&GRACEFSRadialFunction::radbase, this, this->rad_lamba, + this->rcut, this->dcut, + this->radbasename, + _1, 0, 0),//update gr, dgr + gr.get_data(), + dgr.get_data(), deltaSplineBins, this->rcut); + + splines_rnl.setupSplines(fr.get_size(), + std::bind(&GRACEFSRadialFunction::all_radfunc, this, _1), // update fr(nr, l), dfr(nr, l) + fr.get_data(), + dfr.get_data(), deltaSplineBins, rcut); +} + +void GRACEFSRadialFunction::evaluate(DOUBLE_TYPE r, SPECIES_TYPE mu_i, SPECIES_TYPE mu_j) { + splines_gk.calcSplines(r); + for (NS_TYPE nr = 0; nr < nradbasemax; nr++) { + gr(nr) = splines_gk.values(nr); + dgr(nr) = splines_gk.derivatives(nr); + } + + splines_rnl.calcSplines(r); + // copy in flatten format + for (size_t ind = 0; ind < fr.get_size(); ind++) { + fr.get_data(ind) = splines_rnl.values.get_data(ind); + dfr.get_data(ind) = splines_rnl.derivatives.get_data(ind); + } + + //multiply by Z + for (int n = 0; n < nradmax; n++) { + for (int l = 0; l <= lmax; l++) { + fr(n, l) *= Z(mu_j, n); + dfr(n, l) *= Z(mu_j, n); + } + } + + //TODO: hardcore repulsion +} \ No newline at end of file diff --git a/lib/ace/src/ace/grace_fs_evaluator.h b/lib/ace/src/ace/grace_fs_evaluator.h new file mode 100644 index 0000000..8c7cf72 --- /dev/null +++ b/lib/ace/src/ace/grace_fs_evaluator.h @@ -0,0 +1,308 @@ +// +// Created by Yury Lysogorskiy on 24.06.24 +// + +#ifndef GRACE_FS_EVALUATOR_H +#define GRACE_FS_EVALUATOR_H + +#include "ace-evaluator/ace_arraynd.h" +#include "ace-evaluator/ace_array2dlm.h" +#include "ace-evaluator/ace_complex.h" +#include "ace-evaluator/ace_timing.h" +#include "ace-evaluator/ace_types.h" +#include "ace-evaluator/ace_radial.h" +#include "ace-evaluator/ace_spherical_cart.h" + +#include + +#include "yaml-cpp/yaml.h" + +struct GRACEFSBasisFunction { + SPECIES_TYPE mu0 = 0; + + RANK_TYPE rank = 0; + DENSITY_TYPE ndensity = 0; + short num_ms_combs = 0; + + NS_TYPE ns; + Array1D ls; // [rank] + Array1D ms_combs; // size = num_ms_combs * rank, effective shape: [num_ms_combs][rank] + Array1D gen_cgs; // [num_ms_combs] + Array1D coeff; // [ndensity] + + void from_YAML(YAML_PACE::Node); + void print() const; + +}; + +struct GRACEFSEmbeddingSpecification { + DENSITY_TYPE ndensity; + vector FS_parameters; ///< parameters for cluster functional, see Eq.(3) in implementation notes or Eq.(53) in PRB 99, 014104 (2019) + string type = "FinnisSinclairShiftedScaled"; ///< FS and embedding function combination + + void from_YAML(YAML_PACE::Node emb_yaml); +}; + +class GRACEFSRadialFunction { +public: + SPECIES_TYPE nelemets; + + NS_TYPE nradmax; + LS_TYPE lmax; + NS_TYPE nradbasemax; + + string radbasename; + + DOUBLE_TYPE rad_lamba; + DOUBLE_TYPE deltaSplineBins = 0.001; + Array3D crad;///< crad_nlk order: [n=0..nradmax-1][l=0..lmax][k=0..nradbase-1] + Array2D Z; // chemical embedding Z [nelements][nradmax] + + vector crad_shape; + vector Z_shape; + + DOUBLE_TYPE rcut; + DOUBLE_TYPE dcut; + + DOUBLE_TYPE rcut_in = 0; + DOUBLE_TYPE dcut_in = 0; + + void from_YAML(YAML_PACE::Node bond_yaml); + + // Arrays for look-up tables. + SplineInterpolator splines_gk; + SplineInterpolator splines_rnl; + + /** + Arrays to store radial functions. + */ + // g_k(r) functions, shape: [nradbase] + Array1D gr = Array1D("gr"); + // derivatives of g_k(r) functions, shape: [nradbase] + Array1D dgr = Array1D("dgr"); + + // R_nl(r) functions, shape: [nradial][lmax+1] + Array2D fr = Array2D("fr"); + // derivatives of R_nl(r) functions, shape: [nradial][lmax+1] + Array2D dfr = Array2D("dfr"); + + void init(); + + void setuplookupRadspline(); + + void evaluate(DOUBLE_TYPE r, SPECIES_TYPE mu_i, SPECIES_TYPE mu_j); + + void simplified_bessel(DOUBLE_TYPE rc, DOUBLE_TYPE x); + + void radbase(DOUBLE_TYPE lam, DOUBLE_TYPE cut, DOUBLE_TYPE dcut, string radbasename, DOUBLE_TYPE r, + DOUBLE_TYPE cut_in, DOUBLE_TYPE dcut_in); + + void all_radfunc(DOUBLE_TYPE r); + + void radfunc(); +}; + +class GRACEFSBasisSet { +public: + SPECIES_TYPE nelements = 0; ///< number of elements in basis set + RANK_TYPE rankmax = 0; ///< maximum value of rank + DENSITY_TYPE ndensitymax = 0; ///< maximum number of densities \f$ \rho^{(p)} \f$ + NS_TYPE nradbase = 0; ///< maximum number of radial \f$\textbf{basis}\f$ function \f$ g_{k}(r) \f$ + LS_TYPE lmax = 0; ///< \f$ l_\textrm{max} \f$ - maximum value of orbital moment \f$ l \f$ + NS_TYPE nradmax = 0; ///< maximum number \f$ n \f$ of radial function \f$ R_{nl}(r) \f$ + DOUBLE_TYPE cutoffmax = 0; ///< maximum value of cutoff distance among all species in basis set + DOUBLE_TYPE deltaSplineBins = 0; ///< Spline interpolation density + vector elements_name; ///< Array of elements name for mapping from index (0..nelements-1) to element symbol (string) + map elements_to_index_map; + string filename; + int max_dB_array_size; + DOUBLE_TYPE nnorm = 1.0; + + GRACEFSEmbeddingSpecification embedding_specifications; + + ACECartesianSphericalHarmonics spherical_harmonics; + GRACEFSRadialFunction radial_functions; + + vector> basis; // [nelements][nfunctions] + + GRACEFSBasisSet() = default; + + explicit GRACEFSBasisSet(const string &filename); + + void load(const string &filename); + + void FS_values_and_derivatives(Array1D &rhos, DOUBLE_TYPE &value, + Array1D &derivatives, + SPECIES_TYPE mu_i); + + // flatten arrays + + vector> ranks_flatten; //[el][func_ind] + vector> ndensity_flatten; //[el][func_ind] + vector> num_ms_combs_flatten; // [el][func_ind]] + vector> ns_flatten; //[el][func_ind] -> func_ind x ns(1) + vector> ls_flatten; //[el][func_ind][rank] -> func_ind x ls(rank) + vector> ms_combs_flatten; // [el][func_ind][num_ms_comb][rank] -> func_ind x num_ms_comb x rank + + vector> gen_cgs_flatten; // [el][func_ind][num_ms_comb] -> func_ind x num_ms_comb x rank + vector> coeff_flatten; //[el][func_ind][n_density] + + vector> func_ind_to_ls; + vector> func_ind_to_ms_combs; + vector> func_ind_to_gen_cgs; + vector> func_ind_to_coeff; + + // per-atom E0s, global shift, global scale + vector E0_shift; + DOUBLE_TYPE shift=0.0; + DOUBLE_TYPE scale=1.0; + + void convert_to_flatten_arrays(); + void print_functions(); + + SPECIES_TYPE get_species_index_by_name(const string &elemname) { + for (SPECIES_TYPE t = 0; t < nelements; t++) { + if (this->elements_name[t] == elemname) + return t; + } + return -1; + } +}; + +class GRACEFSBEvaluator { +public: + ///3D array with (l,m) last indices for storing A's for rank>1: A(n, l, m) + Array3DLM A = Array3DLM("A"); + + Array1D rhos = Array1D( + "rhos"); ///< densities \f$ \rho^{(p)} \f$(ndensity), p = 0 .. ndensity-1 + Array1D dF_drho = Array1D( + "dF_drho"); ///< derivatives of cluster functional wrt. densities, index = 0 .. ndensity-1 + + /** + * Mapping from external atoms types, i.e. LAMMPS, to internal SPECIES_TYPE, used in basis functions + */ + Array1D element_type_mapping = Array1D("element_type_mapping"); + + DOUBLE_TYPE e_atom = 0; ///< energy of current atom, including core-repulsion + + /** + * The key method to compute energy and forces for atom 'i'. + * Method will update the "e_atom" variable and "neighbours_forces(jj, alpha)" array + * + * @param i atom index + * @param x atomic positions array of the real and ghost atoms, shape: [atom_ind][3] + * @param type atomic types array of the real and ghost atoms, shape: [atom_ind] + * @param jnum number of neighbours of atom_i + * @param jlist array of neighbour indices, shape: [jnum] + */ + void compute_atom(int i, DOUBLE_TYPE **x, const SPECIES_TYPE *type, const int jnum, const int *jlist); + + /** + * Resize all caches over neighbours atoms + * @param max_jnum maximum number of neighbours + */ + void resize_neighbours_cache(int max_jnum); + +#ifdef EXTRA_C_PROJECTIONS + bool compute_projections = false; + /* 1D array to store projections of basis function (all ranks), shape: [func_ind] */ + Array1D projections = Array1D("projections"); + +// Array1D dE_dc = Array1D("dE_dc"); + + // active sets + map> A_active_set_inv; + + DOUBLE_TYPE max_gamma_grade = 0; + + void load_active_set(const string &asi_filename); +#endif + + /** + * Weights \f$ \omega_{i \mu n l m} \f$ for rank > 1, see Eq.(10) from implementation notes, + * 'i' is fixed for the current atom, shape: [nelements][nradbase][l=0..lmax, m] + */ + Array3DLM weights = Array3DLM("weights"); + + + /** + * cache for gradients of \f$ g(r)\f$: grad_phi(jj,n)=A2DLM(l,m) + * shape:[max_jnum][nradbase] + */ + Array2D DG_cache = Array2D("DG_cache"); + + /** + * cache for \f$ R_{nl}(r)\f$ + * shape:[max_jnum][nradbase][0..lmax] + */ + Array3D R_cache = Array3D("R_cache"); + /** + * cache for derivatives of \f$ R_{nl}(r)\f$ + * shape:[max_jnum][nradbase][0..lmax] + */ + Array3D DR_cache = Array3D("DR_cache"); + /** + * cache for \f$ Y_{lm}(\hat{r})\f$ + * shape:[max_jnum][0..lmax][m] + */ + Array3DLM Y_cache = Array3DLM("Y_cache"); + /** + * cache for \f$ \nabla Y_{lm}(\hat{r})\f$ + * shape:[max_jnum][0..lmax][m] + */ + Array3DLM DY_cache = Array3DLM("dY_dense_cache"); + + /** + * cache for derivatives of hard-core repulsion + * shape:[max_jnum] + */ + Array1D DCR_cache = Array1D("DCR_cache"); + + /** + * Partial derivatives \f$ dB_{i \mu n l m t}^{(r)} \f$ with sequential numbering over [func_ind][ms_ind][r], + * shape:[func_ms_r_ind] + */ + Array1D dB_flatten = Array1D("dB_flatten"); + Array1D r_norms = Array1D("r_norms"); + Array1D inv_r_norms = Array1D("inv_r_norms"); + Array2D rhats = Array2D("rhats");//normalized vector + Array1D elements = Array1D("elements"); + + GRACEFSBasisSet basis_set; + + + /** + * temporary array for the pair forces between current atom_i and its neighbours atom_k + * neighbours_forces(k,3), k = 0..num_of_neighbours(atom_i)-1 + */ + Array2D neighbours_forces = Array2D("neighbours_forces"); + + /** + * Initialize internal arrays according to basis set sizes + * @param basis_set + */ + void init(GRACEFSBasisSet &basis_set); + + void set_basis(GRACEFSBasisSet &basis_set); + + ACETimer setup_timer; + ACETimer A_construction_timer; ///< timer for loop over neighbours when constructing A's for single central atom + ACETimer A_construction_timer2; ///< timer for loop over neighbours when constructing A's for single central atom + ACETimer weights_and_theta_timer; + ACETimer per_atom_calc_timer; ///< timer for single compute_atom call + + + ACETimer forces_calc_loop_timer; ///< timer for forces calculations for single central atom +// ACETimer forces_calc_neighbour_timer; ///< timer for loop over neighbour atoms for force calculations + + ACETimer energy_calc_timer; ///< timer for energy calculation + ACETimer total_time_calc_timer; ///< timer for total calculations of all atoms within given atomic environment system + + /** + * Initialize all timers + */ + void init_timers(); +}; + +#endif //GRACE_FS_EVALUATOR_H diff --git a/lib/ace/src/extra/grace_fs_calculator.cpp b/lib/ace/src/extra/grace_fs_calculator.cpp new file mode 100644 index 0000000..6e417fa --- /dev/null +++ b/lib/ace/src/extra/grace_fs_calculator.cpp @@ -0,0 +1,142 @@ +// +// Created by Yury Lysogorskiy on 24.06.24 +// +#include "grace_fs_calculator.h" + +void GRACEFSCalculator::compute(ACEAtomicEnvironment &atomic_environment, bool compute_projections, + bool verbose) { + evaluator.init_timers(); + evaluator.total_time_calc_timer.start(); + +#ifdef EXTRA_C_PROJECTIONS + evaluator.compute_projections = compute_projections; +#endif + int i, j, jj; + double fx, fy, fz, dx, dy, dz; + + energy = 0; + + + energies.resize(atomic_environment.n_atoms_real); + energies.fill(0); + forces.resize(atomic_environment.n_atoms_real, 3);// per-atom forces + forces.fill(0); + + virial.fill(0); + + + //loop over atoms + //determine the maximum number of neighbours + int max_jnum = 0; + for (i = 0; i < atomic_environment.n_atoms_real; ++i) + if (atomic_environment.num_neighbours[i] > max_jnum) + max_jnum = atomic_environment.num_neighbours[i]; + + evaluator.resize_neighbours_cache(max_jnum); +#ifdef EXTRA_C_PROJECTIONS + if (evaluator.compute_projections) { + projections.resize(atomic_environment.n_atoms_real); +// rhos.resize(atomic_environment.n_atoms_real); +// dF_drhos.resize(atomic_environment.n_atoms_real); +// dE_dc.resize(atomic_environment.n_atoms_real); + gamma_grade.resize(atomic_environment.n_atoms_real); + } +#endif + for (i = 0; i < atomic_environment.n_atoms_real; ++i) { + evaluator.compute_atom(i, + atomic_environment.x, + atomic_environment.species_type, + atomic_environment.num_neighbours[i], + atomic_environment.neighbour_list[i]); + //this will also update the e_atom and neighbours_forces(jj, alpha) array +#ifdef EXTRA_C_PROJECTIONS + if (evaluator.compute_projections) { + projections[i] = evaluator.projections.to_vector(); + gamma_grade[i] = evaluator.max_gamma_grade; + } +#endif + //update global energies and forces accumulators + energies(i) = evaluator.e_atom; + + energy += evaluator.e_atom; + + + const DOUBLE_TYPE xtmp = atomic_environment.x[i][0]; + const DOUBLE_TYPE ytmp = atomic_environment.x[i][1]; + const DOUBLE_TYPE ztmp = atomic_environment.x[i][2]; + + for (jj = 0; jj < atomic_environment.num_neighbours[i]; jj++) { + j = atomic_environment.neighbour_list[i][jj]; + + dx = atomic_environment.x[j][0] - xtmp; + dy = atomic_environment.x[j][1] - ytmp; + dz = atomic_environment.x[j][2] - ztmp; + + fx = evaluator.neighbours_forces(jj, 0); + fy = evaluator.neighbours_forces(jj, 1); + fz = evaluator.neighbours_forces(jj, 2); + + forces(i, 0) += fx; + forces(i, 1) += fy; + forces(i, 2) += fz; + + + //virial f_dot_r, identical to LAMMPS virial_fdotr_compute + virial(0) += dx * fx; + virial(1) += dy * fy; + virial(2) += dz * fz; + virial(3) += dx * fy; + virial(4) += dx * fz; + virial(5) += dy * fz; + + // update forces only for real atoms + if (j < atomic_environment.n_atoms_real) { + forces(j, 0) -= fx; + forces(j, 1) -= fy; + forces(j, 2) -= fz; + + } else if (atomic_environment.origins != nullptr) { // map ghost j into true_j within periodic cell + int true_j = atomic_environment.origins[j]; + if (true_j > atomic_environment.n_atoms_real) + throw invalid_argument( + "Inconsistency of atomic environment: origin index j = " + to_string(true_j) + + "out of real atom index range"); + forces(true_j, 0) -= fx; + forces(true_j, 1) -= fy; + forces(true_j, 2) -= fz; + } else { + throw invalid_argument( + "Atomic environment is not consistent: no origins array for mapping ghost atoms"); + } + } + } // loop over atoms (i_at) + + evaluator.total_time_calc_timer.stop(); + +#ifdef FINE_TIMING + if (verbose) { + printf("(Calculator)Total time: %ld microseconds\n", evaluator.total_time_calc_timer.as_microseconds()); + printf("(Evaluator) Total time/at: %ld microseconds\n", + evaluator.per_atom_calc_timer.as_microseconds() / atomic_environment.n_atoms_real); + + printf("setup/atom: %ld microseconds\n", + evaluator.setup_timer.as_microseconds() / atomic_environment.n_atoms_real); + + printf("A_construction/atom: %ld microseconds\n", + evaluator.A_construction_timer.as_microseconds() / atomic_environment.n_atoms_real); + + printf("A_construction2/atom: %ld microseconds\n", + evaluator.A_construction_timer2.as_microseconds() / atomic_environment.n_atoms_real); + + printf("Energy/atom: %ld microseconds\n", + evaluator.energy_calc_timer.as_microseconds() / atomic_environment.n_atoms_real); + printf("Weights and theta/atom: %ld microseconds\n", + evaluator.weights_and_theta_timer.as_microseconds() / atomic_environment.n_atoms_real); + printf("Forces/atom: %ld microseconds\n", + evaluator.forces_calc_loop_timer.as_microseconds() / atomic_environment.n_atoms_real); + } +#endif + + +} + diff --git a/lib/ace/src/extra/grace_fs_calculator.h b/lib/ace/src/extra/grace_fs_calculator.h new file mode 100644 index 0000000..0c4cfdf --- /dev/null +++ b/lib/ace/src/extra/grace_fs_calculator.h @@ -0,0 +1,42 @@ +// +// Created by Yury Lysogorskiy on 31.05.23. +// + +#ifndef GRACE_FS_CALCULATOR_H +#define GRACE_FS_CALCULATOR_H + +#include "ace/grace_fs_evaluator.h" +#include "extra/ace_atoms.h" + +class GRACEFSCalculator { + GRACEFSBEvaluator &evaluator; +public: + //total energy of ACEAtomicEnvironment + DOUBLE_TYPE energy = 0; + //total forces array + //forces(i,3), i = 0..num_of_atoms-1 + Array2D forces = Array2D("forces"); + + //stresses + Array1D virial = Array1D(6, "virial"); + + //Per-atom energies + //energies(i), i = 0..num_of_atoms-1 + Array1D energies = Array1D("energies"); + + explicit GRACEFSCalculator(GRACEFSBEvaluator &aceEvaluator) : evaluator(aceEvaluator) {}; + + + //compute the energies and forces for each atoms in atomic_environment + //results are stored in forces and energies arrays + void compute(ACEAtomicEnvironment &atomic_environment, bool compute_projections = false, bool verbose = false); +#ifdef EXTRA_C_PROJECTIONS + vector> projections; +// vector> rhos; +// vector> dF_drhos; +// vector> dE_dc; + vector gamma_grade; +#endif +}; + +#endif //GRACE_FS_CALCULATOR_H diff --git a/lib/ace/test/fitting/potentials/FS_model_shift_scale.asi b/lib/ace/test/fitting/potentials/FS_model_shift_scale.asi new file mode 100644 index 0000000000000000000000000000000000000000..02dbc417edaea054d9c60e486dd1c51ec7d80f62 GIT binary patch literal 2116 zcmd6oXH*kb6vxAosj|zm1ZBhsGy)=3Avap62qe%3WEKbnBB+GnVu?7Q0gjB~KtV8q zGz^i$vI@K)(+UC#Wyla^2wKZfkODT5({q4x`lX+|@4WZUyZ8M+{qFf+TPq=9dCYQX zU?vG$#WDpj82(;|k;m8s=pSXC62M4*&hUTnEBr_y+uPVW31E(6!gRdp!CpZ+yRbTY zd`xt7u{u5hK_Nk&N8JO0yy^VBh3Da5Ixip0@MO|?Z=A6%)_AKfHUj&;~#Y{5P{fD{!q+u9ahR;%HIRD9Tai1g(4*r_XIXm}^yaDs}1*m>C^_v7wJaHd-; zROwg>5t5h!xfpej(!1+h1uwk)l zHla8IeoB{rZ_}RzcTSxi!kguw<7}7_A524gee;xBTj9keW_eE_tZ6Mr-`BNI*%m3QbRN^sCI&3hKG;qawy8P zc<60<8J}CBiWX1=f%<79sIw2&?Yz1~G_=hBrm-&wisr5j)N9(K;XWf_TGDRFCQ8SY zMw+0OP9Oc?3zbnl>er>hK=xLYe}Q@`61#;T zlN578Z|Q_rcPC7dr1(J8s8%=ZO(AA4iPj<;SHZ`*_+p@$cNmN{6$8eEEwMTW70Ay9YOC{jw)u}=jH^WE?$58g)|8>s z>5R7>N3f({6b+lBTx7X(W~I!@Od(R5BvEujjtEJe9==XgN}IcyYmJ>p>`Jaw&o7m0 z?}EmJ$OmNeN?yAVGWFl8RY*UfS z1u({QHiZwKK@?nab8$>8gg!mWj*Jh5?yCup*6&D0(`k9zQ*V|)tRsH-XNxRIWb9u& z>(&B?oXvI;B|0lx~m z6x6<{MT66EF=tBZ(Xpvszw=p5@NshnC8=r&LHgN$2o=OEwB?xT@})9{QK72+xNPeyLbxV@`4;y~KgjB5dlYM?{T^DGZJ@ z4sssq?y~(82X$f!_FwPR1FHwReW9*f(bK^56Hb9-q&IVqbLWE`{5T@XiWG5!0?R6y z!`^g6X9(@tQR9o+C?3xvx@pjP|E>sO&JF0BZj6Vir2wTiem|op8qQtJC{>?00}nPA zoF%AbqMnFFaYpKCO!o2l_ce{)7U;hAc9qB0m literal 0 HcmV?d00001 diff --git a/lib/ace/test/fitting/potentials/FS_model_shift_scale.yaml b/lib/ace/test/fitting/potentials/FS_model_shift_scale.yaml new file mode 100644 index 0000000..d9799a3 --- /dev/null +++ b/lib/ace/test/fitting/potentials/FS_model_shift_scale.yaml @@ -0,0 +1,41 @@ +# /home/users/lysogy36/acefit/GRACE/TDACE/FS-export/seed/1 +elements: [Mo, Nb, Ta, W] +nnorm: 0.0173492815879129 +emb_spec: {params: [1.0, 1.0, 1.0, 0.5], type: FinnisSinclairShiftedScaled} +radial_basis: {Z: [0.9309980452340629, 1.2614922977371636, -0.3272906611757781, 0.71165300140449, + -0.30123845101606994, 0.5228308972691108, 0.6499629861480888, 0.6284436482387074], + Z_shape: [4, 2], crad: [0.313936681771871, 1.0168338924813647, -0.9811623368752479, + 0.3617097420000017], crad_shape: [2, 1, 2], cutoff: 6, n_rad_base: 2, nradmax: 2, + radbasename: SBessel} +functions: + 0: + - {mu0: 0, ns: 1, ls: [0], ms_combs: [0], gen_cgs: [1.0], ndensity: 2, coeff: [0.08563044921215866, 0.4470457691704677], rank: 1} + - {mu0: 0, ns: 2, ls: [0], ms_combs: [0], gen_cgs: [1.0], ndensity: 2, coeff: [0.47241803566450546, 0.20287804868484796], rank: 1} + - {mu0: 0, ns: 1, ls: [0, 0], ms_combs: [0, 0], gen_cgs: [1.0], ndensity: 2, coeff: [-0.0238811943687232, 1.3671608745574546], rank: 2} + - {mu0: 0, ns: 2, ls: [0, 0], ms_combs: [0, 0], gen_cgs: [1.0], ndensity: 2, coeff: [-0.5103439179294194, -0.18711320109127436], rank: 2} + - {mu0: 0, ns: 1, ls: [0, 0, 0], ms_combs: [0, 0, 0], gen_cgs: [1.0], ndensity: 2, coeff: [-0.5324182968359121, 1.171886633791288], rank: 3} + - {mu0: 0, ns: 2, ls: [0, 0, 0], ms_combs: [0, 0, 0], gen_cgs: [1.0], ndensity: 2, coeff: [-0.7382390734480386, 0.632059294460949], rank: 3} + 1: + - {mu0: 1, ns: 1, ls: [0], ms_combs: [0], gen_cgs: [1.0], ndensity: 2, coeff: [-0.3637024652311783, 0.5476963591511613], rank: 1} + - {mu0: 1, ns: 2, ls: [0], ms_combs: [0], gen_cgs: [1.0], ndensity: 2, coeff: [0.9081935283110572, 1.5968790425657338], rank: 1} + - {mu0: 1, ns: 1, ls: [0, 0], ms_combs: [0, 0], gen_cgs: [1.0], ndensity: 2, coeff: [0.031405552895863616, 2.0966704069954747], rank: 2} + - {mu0: 1, ns: 2, ls: [0, 0], ms_combs: [0, 0], gen_cgs: [1.0], ndensity: 2, coeff: [1.0485997599344958, 0.16707480013544004], rank: 2} + - {mu0: 1, ns: 1, ls: [0, 0, 0], ms_combs: [0, 0, 0], gen_cgs: [1.0], ndensity: 2, coeff: [-0.998818334244998, 1.9914104468306968], rank: 3} + - {mu0: 1, ns: 2, ls: [0, 0, 0], ms_combs: [0, 0, 0], gen_cgs: [1.0], ndensity: 2, coeff: [0.225415627926786, -1.6118200485179413], rank: 3} + 2: + - {mu0: 2, ns: 1, ls: [0], ms_combs: [0], gen_cgs: [1.0], ndensity: 2, coeff: [0.7227341673751824, -0.5637645841986675], rank: 1} + - {mu0: 2, ns: 2, ls: [0], ms_combs: [0], gen_cgs: [1.0], ndensity: 2, coeff: [0.9024895219543086, 0.17634768344374863], rank: 1} + - {mu0: 2, ns: 1, ls: [0, 0], ms_combs: [0, 0], gen_cgs: [1.0], ndensity: 2, coeff: [2.5217106507831772, -0.44086480135819617], rank: 2} + - {mu0: 2, ns: 2, ls: [0, 0], ms_combs: [0, 0], gen_cgs: [1.0], ndensity: 2, coeff: [-0.0007691526951637052, 1.1815139330564068], rank: 2} + - {mu0: 2, ns: 1, ls: [0, 0, 0], ms_combs: [0, 0, 0], gen_cgs: [1.0], ndensity: 2, coeff: [0.34349116954726977, 0.4378611844786709], rank: 3} + - {mu0: 2, ns: 2, ls: [0, 0, 0], ms_combs: [0, 0, 0], gen_cgs: [1.0], ndensity: 2, coeff: [-1.2071819898589273, 2.9196344982236653], rank: 3} + 3: + - {mu0: 3, ns: 1, ls: [0], ms_combs: [0], gen_cgs: [1.0], ndensity: 2, coeff: [-0.3139632951976988, 0.7800881603096272], rank: 1} + - {mu0: 3, ns: 2, ls: [0], ms_combs: [0], gen_cgs: [1.0], ndensity: 2, coeff: [-0.8685178025900058, -0.9853245760471054], rank: 1} + - {mu0: 3, ns: 1, ls: [0, 0], ms_combs: [0, 0], gen_cgs: [1.0], ndensity: 2, coeff: [-0.6179090607443187, -1.2818284282532995], rank: 2} + - {mu0: 3, ns: 2, ls: [0, 0], ms_combs: [0, 0], gen_cgs: [1.0], ndensity: 2, coeff: [-0.266852502045759, 0.3637979134713648], rank: 2} + - {mu0: 3, ns: 1, ls: [0, 0, 0], ms_combs: [0, 0, 0], gen_cgs: [1.0], ndensity: 2, coeff: [-1.1724793581834139, 0.10193739858088074], rank: 3} + - {mu0: 3, ns: 2, ls: [0, 0, 0], ms_combs: [0, 0, 0], gen_cgs: [1.0], ndensity: 2, coeff: [1.3735679398167813, -0.8303176534204201], rank: 3} +E0: [-10.881599725662017, -10.146458813092114, -11.594690153361158, -12.73933999408068] +shift: 0.0 +scale: 0.8142895310895377 diff --git a/setup.py b/setup.py index c35dadb..1ed9c2d 100644 --- a/setup.py +++ b/setup.py @@ -186,18 +186,19 @@ def build_extension(self, ext: CMakeExtension) -> None: CMakeExtension('pyace/evaluator', target='evaluator'), CMakeExtension('pyace/catomicenvironment', target='catomicenvironment'), CMakeExtension('pyace/calculator', target='calculator'), + CMakeExtension('pyace/grace_fs', target='grace_fs'), ], # add custom build_ext command cmdclass=versioneer.get_cmdclass(dict(install=InstallMaxVolPyLocalPackage, build_ext=CMakeBuild)), zip_safe=False, url='https://github.com/ICAMS/python-ace', - install_requires=['numpy<=1.26.4', + install_requires=['numpy<2.0.0', 'ase', - 'pandas<=2.0', + 'pandas', 'ruamel.yaml', 'psutil', - 'scikit-learn<=1.4.2' + 'scikit-learn' ], classifiers=[ 'Programming Language :: Python :: 3', diff --git a/src/pyace/ace/grace_fs_binding.cpp b/src/pyace/ace/grace_fs_binding.cpp new file mode 100644 index 0000000..e55f757 --- /dev/null +++ b/src/pyace/ace/grace_fs_binding.cpp @@ -0,0 +1,101 @@ +// +// Created by Lysogorskiy Yury on 28.06.24. +// + + +#include +#include +#include +#include +#include + +#include "ace/grace_fs_evaluator.h" +#include "extra/grace_fs_calculator.h" + +namespace py = pybind11; +using namespace std; + +PYBIND11_MODULE(grace_fs, m) { + py::options options; + options.disable_function_signatures(); + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// GRACEFSBasisSet +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + py::class_(m, "GRACEFSBasisSet") + .def(py::init<>()) + .def(py::init(), py::arg("yaml_file_name")) + .def_readonly("nelements", &GRACEFSBasisSet::nelements) + .def_property_readonly("elements_name", [](const GRACEFSBasisSet &b) { + vector res(b.nelements); + for (int i = 0; i < b.nelements; i++) + res[i] = b.elements_name[i]; + return res; + }) + .def_readonly("elements_to_index_map", &GRACEFSBasisSet::elements_to_index_map) + .def_readonly("ndensitymax", &GRACEFSBasisSet::ndensitymax) + .def_readonly("nradbase", &GRACEFSBasisSet::nradbase) + .def_readonly("lmax", &GRACEFSBasisSet::lmax) + .def_readonly("nradmax", &GRACEFSBasisSet::nradmax) + .def_readonly("cutoffmax", &GRACEFSBasisSet::cutoffmax) + .def_property_readonly("nfuncs", [](const GRACEFSBasisSet &b) { + vector res(b.nelements); + for (int i = 0; i < b.nelements; i++) + res[i] = b.basis[i].size(); + return res; + }); + + + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// GRACEFSBEvaluator +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +//base class bindings + py::class_(m, "GRACEFSBEvaluator") + .def_property("element_type_mapping", + [](const GRACEFSBEvaluator &e) { return e.element_type_mapping.to_vector(); }, + [](GRACEFSBEvaluator &e, vector v) { e.element_type_mapping = v; }) + .def(py::init<>()) + .def("set_basis", &GRACEFSBEvaluator::set_basis, R"delim( + + Set a basis to the evaluator + + Parameters + ---------- + basis : GRACEFSBasisSet object + + Returns + ------- + None + + )delim") + .def("load_active_set", &GRACEFSBEvaluator::load_active_set, py::arg("asi_filename")) + ; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// GRACEFSCalculator +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + py::class_(m, "GRACEFSCalculator", R"mydelimiter( + + )mydelimiter") + .def(py::init(), py::arg("GRACEFSCalculator")) +// .def("set_evaluator", &ACECalculator::set_evaluator) + .def("compute", &GRACEFSCalculator::compute, py::arg("atomic_environment"), py::arg("compute_projections") = false, py::arg("verbose") = false) + .def_property_readonly("forces", [](const GRACEFSCalculator &calc) { return calc.forces.to_vector(); }) + .def_readonly("energy", &GRACEFSCalculator::energy) + .def_property_readonly("energies", [](const GRACEFSCalculator &calc) { return calc.energies.to_vector(); }) + .def_property_readonly("virial", [](const GRACEFSCalculator &calc) { return calc.virial.to_vector(); }) + .def_readonly("projections", &GRACEFSCalculator::projections) + .def_readonly("gamma_grade",&GRACEFSCalculator::gamma_grade) +// .def_readonly("rhos", &ACECalculator::rhos) +// .def_readonly("dF_drhos", &ACECalculator::dF_drhos) +// .def_readonly("dE_dc", &ACECalculator::dE_dc) +// .def_readonly("forces_bfuncs",&ACECalculator::forces_bfuncs) + ; +#ifdef VERSION_INFO + m.attr("__version__") = VERSION_INFO; +#else + m.attr("__version__") = "dev"; +#endif +} + diff --git a/src/pyace/activelearning.py b/src/pyace/activelearning.py index d64131f..7355ff8 100644 --- a/src/pyace/activelearning.py +++ b/src/pyace/activelearning.py @@ -2,18 +2,20 @@ import os from collections import Counter, defaultdict +from typing import Dict, List, Optional, Union, Tuple import pandas as pd from ase import Atoms from ase.io.lammpsrun import read_lammps_dump_text from maxvolpy.maxvol import maxvol -from pyace.asecalc import PyACECalculator +from pyace.asecalc import PyACECalculator, PyGRACEFSCalculator from pyace.atomicenvironment import aseatoms_to_atomicenvironment, ACEAtomicEnvironment from pyace.basis import BBasisConfiguration, ACEBBasisSet from pyace.calculator import ACECalculator from pyace.evaluator import ACEBEvaluator -from typing import Dict, List, Optional, Union, Tuple + +from pyace.grace_fs import GRACEFSBasisSet, GRACEFSBEvaluator, GRACEFSCalculator def simple_tqdm(iterable, **kwargs): @@ -89,18 +91,19 @@ def count_number_total_atoms_per_species_type_by_batches(atomic_env_batches: Lis return dict(n_total_atoms_per_species_type) -def compute_B_projections(bconf: Union[BBasisConfiguration, ACEBBasisSet, PyACECalculator], - atomic_env_list: Union[List[Atoms], List[ACEAtomicEnvironment]], - structure_ind_list: Optional[List[int]] = None, - is_full=False, - compute_forces_dict=False, - return_structure_ind_dict=False, - verbose=False - ) -> Tuple[Dict[int, np.array], Optional[Dict[int, np.array]]]: +def compute_B_projections( + bconf: Union[BBasisConfiguration, ACEBBasisSet, PyACECalculator, GRACEFSCalculator, GRACEFSBasisSet], + atomic_env_list: Union[List[Atoms], List[ACEAtomicEnvironment]], + structure_ind_list: Optional[List[int]] = None, + is_full=False, + compute_forces_dict=False, + return_structure_ind_dict=False, + verbose=False + ) -> Tuple[Dict[int, np.array], Optional[Dict[int, np.array]]]: """ Function to compute the B-basis projection using basis configuration `bconf` for list of atomic environments `atomic_env_list`. - :param bconf: BBasisConfiguration + :param bconf: [BBasisConfiguration, ACEBBasisSet, PyACECalculator, GRACEFSCalculator, GRACEFSBasisSet] :param atomic_env_list: list of ACEAtomicEnvironment or ASE atoms :param structure_ind_list: optional, list of corresponding indices of structures/atomic envs. :param is_full: MaxVol on linear B-projections (false, default) or on full non-linear atomic energy (True) @@ -142,9 +145,13 @@ def compute_B_projections(bconf: Union[BBasisConfiguration, ACEBBasisSet, PyACEC # count total number of atoms of each species type in whole dataset atomiv_env_list n_total_atoms_per_species_type = count_number_total_atoms_per_species_type(atomic_env_list) - beval = ACEBEvaluator(bbasis) - - calc = ACECalculator(beval) + if isinstance(bbasis, ACEBBasisSet): + beval = ACEBEvaluator(bbasis) + calc = ACECalculator(beval) + elif isinstance(bbasis, GRACEFSBasisSet): + beval = GRACEFSBEvaluator() + beval.set_basis(bbasis) + calc = GRACEFSCalculator(beval) # prepare numpy arrays for A0_projections and structure_ind_dict A0_projections_dict = {st: np.zeros((n_total_atoms_per_species_type[st], @@ -191,9 +198,9 @@ def compute_B_projections(bconf: Union[BBasisConfiguration, ACEBBasisSet, PyACEC def convert_to_bbasis(bconf): if isinstance(bconf, BBasisConfiguration): bbasis = ACEBBasisSet(bconf) - elif isinstance(bconf, ACEBBasisSet): + elif isinstance(bconf, (ACEBBasisSet, GRACEFSBasisSet)): bbasis = bconf - elif isinstance(bconf, PyACECalculator): + elif isinstance(bconf, (PyACECalculator, PyGRACEFSCalculator)): bbasis = bconf.basis else: raise ValueError("Unsupported type of `bconf`: {}".format(type(bconf))) @@ -233,7 +240,10 @@ def extract_reference_forces_dict(ase_atoms_list: List[Atoms], def compute_number_of_functions(pot): - return [len(b1) + len(b) for b1, b in zip(pot.basis_rank1, pot.basis)] + if isinstance(pot, ACEBBasisSet): + return [len(b1) + len(b) for b1, b in zip(pot.basis_rank1, pot.basis)] + elif isinstance(pot, GRACEFSBasisSet): + return pot.nfuncs def compute_extrapolation_grade(A0_projections_dict: Dict[int, np.array], A_active_set_inv_dict: Dict[int, np.array]) -> \ @@ -476,7 +486,7 @@ def compute_active_set_by_batches(bbasis: Union[BBasisConfiguration, ACEBBasisSe if structure_ind_list is None: structure_ind_list = np.arange(len(atomic_env_list), dtype=int) - bbasis = convert_to_bbasis(bbasis) + bbasis = convert_to_bbasis(bbasis) # ACEBBasisSet, GRACEFSBasisSet atomic_env_list = convert_to_atomic_env_list(atomic_env_list, bbasis) batch_splits_indices = np.array_split(np.arange(len(atomic_env_list)), n_batches) atomic_env_batches = [atomic_env_list[bsi] for bsi in batch_splits_indices] diff --git a/src/pyace/asecalc.py b/src/pyace/asecalc.py index 359d6f9..a32d738 100644 --- a/src/pyace/asecalc.py +++ b/src/pyace/asecalc.py @@ -1,4 +1,5 @@ import numpy as np +import time from ase.calculators.calculator import Calculator, all_changes from ase.io import write @@ -9,6 +10,8 @@ from pyace.catomicenvironment import ACEAtomicEnvironment from pyace.evaluator import ACEBEvaluator, ACECTildeEvaluator, ACERecursiveEvaluator +from pyace.grace_fs import GRACEFSBasisSet, GRACEFSBEvaluator, GRACEFSCalculator + class PyACECalculator(Calculator): """ @@ -329,3 +332,126 @@ def calculate(self, atoms=None, properties=('energy', 'forces', 'stress', 'energ self.results["stress"] = self.stress self.results["stress_std"] = self.stress_std self.results["stress_dev"] = self.stress_dev + + +# TODO: made this calculator part of PyACECalculator +class PyGRACEFSCalculator(Calculator): + """ + Python ASE calculator wrapper for GRACE/FS C++ native implementation + :param basis_set - ".yaml" potential filename + + """ + implemented_properties = ['energy', 'forces', 'stress', 'energies', 'free_energy'] + + def __init__(self, basis_set, **kwargs): + """ +PyGRACEFSCalculator calculator +:param basis_set - specification of GRACE/FS potential, could be in following forms: + ".yaml" potential filename +""" + if "fast_nl" not in kwargs: + kwargs["fast_nl"] = True + + Calculator.__init__(self, basis_set=basis_set, **kwargs) + self.nl = None + self.skin = 0. + # self.reset_nl = True # Set to False for MD simulations + self.ae = ACEAtomicEnvironment() + + self._create_evaluator() + + self.cutoff = self.basis.cutoffmax # self.parameters.basis_config.funcspecs_blocks[0].rcutij + + self.energy = None + self.energies = None + self.forces = None + self.virial = None + self.stress = None + self.projections = None + self.current_extrapolation_structure_index = 0 + self.is_active_set_configured = False + self.extrapolative_structures_list = [] + self.extrapolative_structures_gamma = [] + self.ace = GRACEFSCalculator(self.evaluator) + # self.ace.set_evaluator() + self.compute_projections = False + + def _create_evaluator(self): + + basis_set = self.parameters.basis_set + if isinstance(basis_set, GRACEFSBasisSet): + self.basis = basis_set + elif isinstance(basis_set, str): + if basis_set.endswith(".yaml"): + self.basis = GRACEFSBasisSet(basis_set) + else: + raise ValueError("Unrecognized file format: " + basis_set) + else: + raise ValueError("Unrecognized basis set specification") + + self.elements_name = np.array(self.basis.elements_name).astype(dtype="S2") + self.elements_mapper_dict = {el: i for i, el in enumerate(self.elements_name)} + + if isinstance(self.basis, GRACEFSBasisSet): + self.evaluator = GRACEFSBEvaluator() + self.evaluator.set_basis(self.basis) + + def get_atomic_env(self, atoms): + try: + if self.parameters.fast_nl: + self.ae = aseatoms_to_atomicenvironment(atoms, cutoff=self.cutoff, + elements_mapper_dict=self.elements_mapper_dict) + else: + self.ae = aseatoms_to_atomicenvironment_old(atoms, cutoff=self.cutoff, + skin=self.skin, + elements_mapper_dict=self.elements_mapper_dict) + except KeyError as e: + raise ValueError("Unsupported species type: " + str(e) + ". Supported elements: " + str(self.elements_name)) + return self.ae + + def calculate(self, atoms=None, properties=('energy', 'forces', 'stress', 'energies'), + system_changes=all_changes): + Calculator.calculate(self, atoms, properties, system_changes) + + self.energy = 0.0 + self.energies = np.zeros(len(atoms)) + self.forces = np.empty((len(atoms), 3)) + + self.get_atomic_env(atoms) + t0 = time.perf_counter() + self.ace.compute(self.ae, compute_projections=self.compute_projections) + self.perf = (time.perf_counter() - t0) + + nat = len(atoms) + try: + self.projections = np.reshape(self.ace.projections, (nat, -1)) + except ValueError: + # if projections has different shapes + self.projections = self.ace.projections + + self.energy, self.forces = np.array(self.ace.energy), np.array(self.ace.forces) + + self.energies = np.array(self.ace.energies) + + self.results = { + 'energy': np.float64(self.energy.reshape(-1, )), + 'free_energy': np.float64(self.energy.reshape(-1, )), + 'forces': self.forces.astype(np.float64), + 'energies': self.energies.astype(np.float64), + 'gamma': np.array(self.ace.gamma_grade, dtype=np.float64) + } + if self.atoms.cell.rank == 3: + self.volume = atoms.get_volume() + self.virial = np.array(self.ace.virial) # order is: xx, yy, zz, xy, xz, yz + # swap order of the virials to fullfill ASE Voigt stresses order: (xx, yy, zz, yz, xz, xy) + self.stress = self.virial[[0, 1, 2, 5, 4, 3]] / self.volume + self.results["stress"] = self.stress + + def set_active_set(self, filename_or_list_of_active_set_inv): + if isinstance(filename_or_list_of_active_set_inv, str): + self.evaluator.load_active_set(filename_or_list_of_active_set_inv) + self.is_active_set_configured = True + self.compute_projections = True + else: + raise ValueError("Unsupported type for `filename_or_list_of_active_set_inv`: {}".format( + type(filename_or_list_of_active_set_inv))) diff --git a/src/pyace/data_aug.py b/src/pyace/data_aug.py index 13d35ad..fe75691 100644 --- a/src/pyace/data_aug.py +++ b/src/pyace/data_aug.py @@ -133,6 +133,7 @@ def calculate( S[d < self.cut_in] = C[d < self.cut_in] S[d > self.cutoff] = 0 + E_ij[d > self.cutoff] = 0 # just to be sure, but NN list is upt to cutoff only self.energy = K / 2 * np.sum(Zi * Zj * (E_ij + S)) # forces @@ -142,6 +143,7 @@ def calculate( dS_dr[d < self.cut_in] = 0 dS_dr[d > self.cutoff] = 0 + dEdr[d > self.cutoff] = 0 # just to be sure, but NN list is upt to cutoff only pair_forces = -(dEdr + dS_dr).reshape(-1, 1) * (D / d.reshape(-1, 1)) pair_forces *= (Zi * Zj).reshape(-1, 1) * K / 2 diff --git a/src/pyace/fitadapter.py b/src/pyace/fitadapter.py index 58b930a..b290f11 100644 --- a/src/pyace/fitadapter.py +++ b/src/pyace/fitadapter.py @@ -168,12 +168,21 @@ def setup_tensorpot(self, bbasisconfig: BBasisConfiguration, dataframe: pd.DataF loss_spec: LossFunctionSpecification, trainable_parameters_dict: Dict ) -> BBasisConfiguration: - from tensorpotential.potentials.ace import ACE - from tensorpotential.tensorpot import TensorPotential - from tensorpotential.fit import FitTensorPotential - from tensorpotential.utils.utilities import batching_data, init_gpu_config - from tensorpotential.constants import (LOSS_TYPE, LOSS_FORCE_FACTOR, LOSS_ENERGY_FACTOR, L1_REG, - L2_REG, AUX_LOSS_FACTOR) + try: + from tensorpotential.compat.pace.potentials.ace import ACE + from tensorpotential.compat.pace.tensorpot import TensorPotential + from tensorpotential.compat.pace.fit import FitTensorPotential + from tensorpotential.compat.pace.utils.utilities import batching_data, init_gpu_config + from tensorpotential.compat.pace.constants import (LOSS_TYPE, LOSS_FORCE_FACTOR, LOSS_ENERGY_FACTOR, L1_REG, + L2_REG, AUX_LOSS_FACTOR) + except ImportError: + from tensorpotential.potentials.ace import ACE + from tensorpotential.tensorpot import TensorPotential + from tensorpotential.fit import FitTensorPotential + from tensorpotential.utils.utilities import batching_data, init_gpu_config + from tensorpotential.constants import (LOSS_TYPE, LOSS_FORCE_FACTOR, LOSS_ENERGY_FACTOR, L1_REG, + L2_REG, AUX_LOSS_FACTOR) + with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=UserWarning) @@ -303,9 +312,14 @@ def setup_backend_for_predict(self, bbasisconfig): raise ValueError("`bbasisconfig` couldn't be None for FitAdapter.setup_backend_for_predict") log.info("Setting {} backend for predicting".format(self.backend_config.evaluator_name)) if self.backend_config.evaluator_name == TENSORPOT_EVAL: - from tensorpotential.potentials.ace import ACE - from tensorpotential.tensorpot import TensorPotential - from tensorpotential.fit import FitTensorPotential + try: + from tensorpotential.compat.pace.potentials.ace import ACE + from tensorpotential.compat.pace.tensorpot import TensorPotential + from tensorpotential.compat.pace.fit import FitTensorPotential + except ImportError: + from tensorpotential.potentials.ace import ACE + from tensorpotential.tensorpot import TensorPotential + from tensorpotential.fit import FitTensorPotential ace_pot = ACE(bbasisconfig) tp = TensorPotential(ace_pot) self.fitter = FitTensorPotential(tensorpot=tp, eager=True) diff --git a/tests/test_calculator.py b/tests/test_calculator.py index 05d7f40..6d2632a 100644 --- a/tests/test_calculator.py +++ b/tests/test_calculator.py @@ -3,7 +3,7 @@ import pytest from ase.atoms import Atoms from ase.build import bulk -from pyace.asecalc import PyACECalculator, PyACEEnsembleCalculator +from pyace.asecalc import PyACECalculator, PyACEEnsembleCalculator, PyGRACEFSCalculator from pyace.atomicenvironment import aseatoms_to_atomicenvironment, create_cube, create_linear_chain from pyace.basis import ACECTildeBasisSet, ACEBBasisSet, FexpShiftedScaled from pyace.calculator import ACECalculator @@ -149,8 +149,6 @@ def test_calculator_bbasis_projections(): print("x: x_projs=", x_projs) - - print("x: x_projs=", x_projs) print("x_rhos=", x_rhos) @@ -448,3 +446,73 @@ def test_PyACECalculator_active_set_dump_extrapolation(): assert len(asecalc.extrapolative_structures_gamma) == 2 assert len(asecalc.extrapolative_structures_list) == 2 + + +def test_GRACEFSCalculator(): + calc = PyGRACEFSCalculator("lib/ace/test/fitting/potentials/FS_model_shift_scale.yaml") + + at = Atoms("Mo3", positions=[[0.0, 0.0, 0.0], [0.0, 0.0, 1.0], [0.0, 0.0, 2.0]], pbc=False) + at.calc = calc + + e = at.get_potential_energy() + f = at.get_forces() + print("calc.results=", calc.results) + print("E=", e, "f=", f[0, 2]) + + eref = -32.70577226767949 + fref = 0.030252962190919178 + energies_ref = np.array([-10.89918829, -10.90739569, -10.89918829]) + + assert np.abs(e - eref) < 1e-15 + assert np.abs(f[0, 2] - fref) < 1e-14 + assert np.allclose(calc.results["energies"], energies_ref, atol=1e-6) + + +def test_GRACEFScalculator_projections(): + calc = PyGRACEFSCalculator("lib/ace/test/fitting/potentials/FS_model_shift_scale.yaml") + calc.compute_projections = True + + at = Atoms("Mo3", positions=[[0.0, 0.0, 0.0], [0.0, 0.0, 1.0], [0.0, 0.0, 2.0]], pbc=False) + at.calc = calc + + e = at.get_potential_energy() + f = at.get_forces() + print("calc.results=", calc.results) + print("E=", e, "f=", f[0, 2]) + + eref = -32.70577226767949 + fref = 0.030252962190919178 + energies_ref = np.array([-10.89918829, -10.90739569, -10.89918829]) + + assert np.abs(e - eref) < 1e-15 + assert np.abs(f[0, 2] - fref) < 1e-14 + assert np.allclose(calc.results["energies"], energies_ref, atol=1e-6) + + print("Projections=", calc.projections) + proj_ref=[-0.007184322195427408, + -0.023008841969645075, + 5.161448540771089e-05, + 0.0005294068087841006, + -3.708150931201814e-07, + -1.218103760096748e-05] + assert np.allclose(calc.projections[0], proj_ref) + +def test_GRACEFScalculator_ASI(): + calc = PyGRACEFSCalculator("lib/ace/test/fitting/potentials/FS_model_shift_scale.yaml") + calc.set_active_set("lib/ace/test/fitting/potentials/FS_model_shift_scale.asi") + + at = Atoms("Mo3", positions=[[0.0, 0.0, 0.0], [0.0, 0.0, 1.0], [0.0, 0.0, 2.0]], pbc=False) + at.calc = calc + + e = at.get_potential_energy() + f = at.get_forces() + print("calc.results=", calc.results) + print("E=", e, "f=", f[0, 2]) + print(calc.results['gamma']) + + + gamma_ref = np.array([1.00645231, 1.73907975, 1.00645231]) + + + assert np.allclose(calc.results["gamma"], gamma_ref, atol=1e-6) + From 38306b92aabaf5bf2f3c97e87b78c740718430c0 Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Thu, 14 Nov 2024 12:21:59 +0100 Subject: [PATCH 02/30] update pace_select for GRACE/FS --- bin/pace_select | 9 ++++++++- src/pyace/aceselect.py | 9 ++++++--- src/pyace/fitadapter.py | 4 ++-- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/bin/pace_select b/bin/pace_select index ce31d02..a572967 100644 --- a/bin/pace_select +++ b/bin/pace_select @@ -8,6 +8,7 @@ import sys from pyace import BBasisConfiguration from pyace.activelearning import compute_A_active_inverse, load_active_inverse_set from pyace.aceselect import load_multiple_datasets, select_structures_maxvol, save_selected_structures +from pyace.grace_fs import GRACEFSBasisSet LOG_FMT = '%(asctime)s %(levelname).1s - %(message)s' logging.basicConfig(level=logging.INFO, format=LOG_FMT, datefmt="%Y/%m/%d %H:%M:%S") @@ -48,7 +49,13 @@ def main(): if verbose: log.info(f"{len(df['ase_atoms'])} structures in candidate list") log.info(f"Loading B-basis configuration from {potential_file}") - bconf = BBasisConfiguration(potential_file) + try: + bconf = BBasisConfiguration(potential_file) + log.info("ACEBBasisSet loaded") + except RuntimeError: + bconf = GRACEFSBasisSet(potential_file) + log.info("GRACEFSBasisSet loaded") + if active_set_inv_fname: if verbose: diff --git a/src/pyace/aceselect.py b/src/pyace/aceselect.py index 33e4e39..3327eb2 100644 --- a/src/pyace/aceselect.py +++ b/src/pyace/aceselect.py @@ -10,8 +10,9 @@ from ase.io import write from pyace import ACEBBasisSet -from pyace.activelearning import compute_B_projections, compute_active_set_by_batches, read_extrapolation_data, \ - compute_number_of_functions, count_number_total_atoms_per_species_type_aseatom, compute_active_set +from pyace.activelearning import (compute_B_projections, compute_active_set_by_batches, read_extrapolation_data, + compute_number_of_functions, count_number_total_atoms_per_species_type_aseatom, compute_active_set, + convert_to_bbasis) from pyace.preparedata import sizeof_fmt LOG_FMT = '%(asctime)s %(levelname).1s - %(message)s' @@ -38,7 +39,9 @@ def select_structures_maxvol(df, bconf, extra_A0_projections_dict=None, batch_si :return: df_selected - subset of input "df" """ - bbasis = ACEBBasisSet(bconf) + # create BBasis configuration + bbasis = convert_to_bbasis(bconf) + nfuncs = compute_number_of_functions(bbasis) elements_to_index_map = bbasis.elements_to_index_map elements_name = bbasis.elements_name diff --git a/src/pyace/fitadapter.py b/src/pyace/fitadapter.py index b290f11..3b1c183 100644 --- a/src/pyace/fitadapter.py +++ b/src/pyace/fitadapter.py @@ -174,14 +174,14 @@ def setup_tensorpot(self, bbasisconfig: BBasisConfiguration, dataframe: pd.DataF from tensorpotential.compat.pace.fit import FitTensorPotential from tensorpotential.compat.pace.utils.utilities import batching_data, init_gpu_config from tensorpotential.compat.pace.constants import (LOSS_TYPE, LOSS_FORCE_FACTOR, LOSS_ENERGY_FACTOR, L1_REG, - L2_REG, AUX_LOSS_FACTOR) + L2_REG, AUX_LOSS_FACTOR) except ImportError: from tensorpotential.potentials.ace import ACE from tensorpotential.tensorpot import TensorPotential from tensorpotential.fit import FitTensorPotential from tensorpotential.utils.utilities import batching_data, init_gpu_config from tensorpotential.constants import (LOSS_TYPE, LOSS_FORCE_FACTOR, LOSS_ENERGY_FACTOR, L1_REG, - L2_REG, AUX_LOSS_FACTOR) + L2_REG, AUX_LOSS_FACTOR) with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=UserWarning) From 39cd17f29c19ff8591cc8c441cee1f0e923e8b68 Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Mon, 23 Feb 2026 14:38:50 +0100 Subject: [PATCH 03/30] bugfix: Replace `pkg_resources` with `importlib.resources` for resource handling. it was breaking since python 3.13 --- bin/pacemaker | 4 ++-- src/pyace/multispecies_basisextension.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/bin/pacemaker b/bin/pacemaker index ed110b4..8bbe02a 100644 --- a/bin/pacemaker +++ b/bin/pacemaker @@ -5,7 +5,7 @@ import getpass # for getpass.getuser() import glob import logging import os -import pkg_resources +from importlib.resources import files import re try: @@ -343,7 +343,7 @@ def generate_template_input(): weighting = None print("Use UniformWeightingPolicy") - template_input_yaml_filename = pkg_resources.resource_filename('pyace.data', 'input_template.yaml') + template_input_yaml_filename = str(files('pyace.data').joinpath('input_template.yaml')) copyfile(template_input_yaml_filename, "input.yaml") with open("input.yaml", "r") as f: input_yaml_text = f.read() diff --git a/src/pyace/multispecies_basisextension.py b/src/pyace/multispecies_basisextension.py index 82bb41e..ea24742 100644 --- a/src/pyace/multispecies_basisextension.py +++ b/src/pyace/multispecies_basisextension.py @@ -1,7 +1,7 @@ import logging import numpy as np import pickle -import pkg_resources +from importlib.resources import files import re from collections import defaultdict @@ -44,8 +44,7 @@ 'Rf', 'Db', 'Sg', 'Bh', 'Hs', 'Mt', 'Ds', 'Rg', 'Cn', 'Nh', 'Fl', 'Mc', 'Lv', 'Ts', 'Og'] -default_mus_ns_uni_to_rawlsLS_np_rank_filename = pkg_resources.resource_filename('pyace.data', - 'mus_ns_uni_to_rawlsLS_np_rank.pckl') +default_mus_ns_uni_to_rawlsLS_np_rank_filename = str(files('pyace.data').joinpath('mus_ns_uni_to_rawlsLS_np_rank.pckl')) def clean_bbasisconfig(initial_bbasisconfig): for block in initial_bbasisconfig.funcspecs_blocks: From 1e69eeb01961e6ba0cacda4692d02f78b605777d Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Mon, 23 Feb 2026 14:41:44 +0100 Subject: [PATCH 04/30] remove test_calculator.py::test_PyACECalculator_active_set_dump_extrapolation --- tests/test_calculator.py | 41 ---------------------------------------- 1 file changed, 41 deletions(-) diff --git a/tests/test_calculator.py b/tests/test_calculator.py index 6d2632a..2670403 100644 --- a/tests/test_calculator.py +++ b/tests/test_calculator.py @@ -405,47 +405,6 @@ def test_PyACECalculator_active_set(): assert np.allclose(gamma, gamma_expected) -def test_PyACECalculator_active_set_dump_extrapolation(): - atoms = bulk("Ag", cubic=True) - atoms.set_chemical_symbols(["Ag", "Cu", "Ag", "Cu"]) - pos = atoms.get_positions() - np.random.seed(42) - pos += np.random.randn(*pos.shape) - atoms.set_positions(pos) - - asecalc = PyACECalculator("tests/DFT10B-AgCu.yaml", - dump_extrapolative_structures=True, - keep_extrapolative_structures=True, - stop_at_large_extrapolation=True) - asecalc.set_active_set("tests/DFT10B-AgCu.asi") - - atoms.set_calculator(asecalc) - - expected_file_name = "extrapolation_0_gamma=5691197.948196411.cfg" - if os.path.isfile(expected_file_name): - os.remove(expected_file_name) - assert not os.path.isfile(expected_file_name) - with pytest.raises(RuntimeError) as excinfo: - energy = atoms.get_potential_energy() - print("Exception raised: ", excinfo) - assert os.path.isfile(expected_file_name) - - # second config - expected_file_name_1 = "extrapolation_1_gamma=5691197.948196411.cfg" - if os.path.isfile(expected_file_name_1): - os.remove(expected_file_name_1) - assert not os.path.isfile(expected_file_name_1) - asecalc.reset() - with pytest.raises(RuntimeError) as excinfo: - energy = atoms.get_potential_energy() - print("Exception raised: ", excinfo) - assert os.path.isfile(expected_file_name_1) - - os.remove(expected_file_name) - os.remove(expected_file_name_1) - - assert len(asecalc.extrapolative_structures_gamma) == 2 - assert len(asecalc.extrapolative_structures_list) == 2 def test_GRACEFSCalculator(): From a7e19fa21037afe66065409a8e66edf5dabdff95 Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Mon, 23 Feb 2026 14:57:44 +0100 Subject: [PATCH 05/30] bump version to numpy>=2.0.0 refactor: Uniformly replace `.set_calculator` with `.calc` and update deprecated warnings --- setup.py | 2 +- src/pyace/activeexploration.py | 6 +++--- src/pyace/asecalc.py | 28 ++++++++++++------------- src/pyace/basisextension.py | 4 ++-- src/pyace/paralleldataexecutor.py | 2 +- src/pyace/radial.py | 2 +- tests/test_ACEAtomicEnvironment.py | 2 +- tests/test_PyACECalculator.py | 33 ++++++++++++++++-------------- tests/test_calculator.py | 10 ++++----- tests/test_generalfit.py | 3 ++- tests/test_linearfit.py | 2 +- 11 files changed, 49 insertions(+), 45 deletions(-) diff --git a/setup.py b/setup.py index 1ed9c2d..84c6f39 100644 --- a/setup.py +++ b/setup.py @@ -193,7 +193,7 @@ def build_extension(self, ext: CMakeExtension) -> None: build_ext=CMakeBuild)), zip_safe=False, url='https://github.com/ICAMS/python-ace', - install_requires=['numpy<2.0.0', + install_requires=['numpy>=2.0.0', 'ase', 'pandas', 'ruamel.yaml', diff --git a/src/pyace/activeexploration.py b/src/pyace/activeexploration.py index 1c74a50..d7321fe 100644 --- a/src/pyace/activeexploration.py +++ b/src/pyace/activeexploration.py @@ -87,7 +87,7 @@ def obj(self, dpos, atom_ind, orig_structure): print("(too small)") return 1000 * (self.min_dist - min_dist) - new_atoms.set_calculator(self.calc) + new_atoms.calc=self.calc self.calc.reset() new_atoms.get_potential_energy() max_gamma = max(self.calc.results["gamma"]) @@ -232,7 +232,7 @@ def active_exploration(self, orig_atoms, # compute gamma wrt. original ASI gat = new_atoms.copy() - gat.set_calculator(self.calc) + gat.calc=self.calc gat.get_potential_energy() gamma = self.calc.results['gamma'] @@ -264,7 +264,7 @@ def active_exploration(self, orig_atoms, return cur_atoms def attach_gamma_array(self, atoms): - atoms.set_calculator(self.calc) + atoms.calc=self.calc self.calc.reset() atoms.get_potential_energy() atoms.arrays["gamma"] = self.calc.results['gamma'] diff --git a/src/pyace/asecalc.py b/src/pyace/asecalc.py index a32d738..1a7b8e3 100644 --- a/src/pyace/asecalc.py +++ b/src/pyace/asecalc.py @@ -167,8 +167,8 @@ def calculate(self, atoms=None, properties=('energy', 'forces', 'stress', 'energ self.energies = np.array(self.ace.energies) self.results = { - 'energy': np.float64(self.energy.reshape(-1, )), - 'free_energy': np.float64(self.energy.reshape(-1, )), + 'energy': float(self.energy), + 'free_energy': float(self.energy), 'forces': self.forces.astype(np.float64), 'energies': self.energies.astype(np.float64), 'gamma': np.array(self.ace.gamma_grade, dtype=np.float64) @@ -276,14 +276,14 @@ def calculate(self, atoms=None, properties=('energy', 'forces', 'stress', 'energ # loop over calculators for calc in self.calcs: cur_atoms = atoms.copy() - cur_atoms.set_calculator(calc) + cur_atoms.calc = calc cur_energy = cur_atoms.get_potential_energy() energy_lst.append(cur_energy) energies_lst.append(cur_atoms.get_potential_energies()) forces_lst.append(cur_atoms.get_forces()) - if self.atoms.number_of_lattice_vectors == 3: + if self.atoms.cell.rank == 3: cur_stress = cur_atoms.get_stress() stress_lst.append(cur_stress) @@ -308,24 +308,24 @@ def calculate(self, atoms=None, properties=('energy', 'forces', 'stress', 'energ self.results = { # mean - 'energy': np.float64(self.energy.reshape(-1, )), - 'free_energy': np.float64(self.energy.reshape(-1, )), + 'energy': float(self.energy), + 'free_energy': float(self.energy), 'forces': self.forces.astype(np.float64), 'energies': self.energies.astype(np.float64), # std - 'energy_std': np.float64(self.energy_std.reshape(-1, )), - 'free_energy_std': np.float64(self.energy_std.reshape(-1, )), + 'energy_std': float(self.energy_std), + 'free_energy_std': float(self.energy_std), 'forces_std': self.forces_std.astype(np.float64), 'energies_std': self.energies_std.astype(np.float64), # dev - 'energy_dev': np.float64(self.energy_dev), - 'energies_dev': np.float64(self.energies_dev), - 'forces_dev': np.float64(self.forces_dev) + 'energy_dev': float(self.energy_dev), + 'energies_dev': self.energies_dev.astype(np.float64), + 'forces_dev': self.forces_dev.astype(np.float64) } - if self.atoms.number_of_lattice_vectors == 3: + if self.atoms.cell.rank == 3: self.stress = np.mean(stress_lst, axis=0) self.stress_std = np.std(stress_lst, axis=0) self.stress_dev = np.max(abs(stress_lst - self.stress), axis=0) @@ -434,8 +434,8 @@ def calculate(self, atoms=None, properties=('energy', 'forces', 'stress', 'energ self.energies = np.array(self.ace.energies) self.results = { - 'energy': np.float64(self.energy.reshape(-1, )), - 'free_energy': np.float64(self.energy.reshape(-1, )), + 'energy': float(self.energy), + 'free_energy': float(self.energy), 'forces': self.forces.astype(np.float64), 'energies': self.energies.astype(np.float64), 'gamma': np.array(self.ace.gamma_grade, dtype=np.float64) diff --git a/src/pyace/basisextension.py b/src/pyace/basisextension.py index f4f75a5..c336307 100644 --- a/src/pyace/basisextension.py +++ b/src/pyace/basisextension.py @@ -98,11 +98,11 @@ def check_backward_compatible_parameters(potential_config: Dict): # "lmax" -> "lmax_by_orders" if POTENTIAL_NRADMAX_KW in potential_config: - log.warn("potential_config:'{}' is deprecated parameter, please use '{}'".format(POTENTIAL_NRADMAX_KW, + log.warning("potential_config:'{}' is deprecated parameter, please use '{}'".format(POTENTIAL_NRADMAX_KW, ORDERS_NRADMAX_KW)) potential_config[ORDERS_NRADMAX_KW] = potential_config[POTENTIAL_NRADMAX_KW] if POTENTIAL_LMAX_KW in potential_config: - log.warn( + log.warning( "potential_config:'{}' is deprecated parameter, please use '{}'".format(POTENTIAL_LMAX_KW, ORDERS_LMAX_KW)) potential_config[ORDERS_LMAX_KW] = potential_config[POTENTIAL_LMAX_KW] diff --git a/src/pyace/paralleldataexecutor.py b/src/pyace/paralleldataexecutor.py index 07d8a78..011cfa9 100644 --- a/src/pyace/paralleldataexecutor.py +++ b/src/pyace/paralleldataexecutor.py @@ -87,7 +87,7 @@ def __init__(self, distributed_data: pd.DataFrame, parallel_mode: str = MODE_SER self.parallel_mode = parallel_mode self._n_workers = n_workers - if self.parallel_mode != ParallelDataExecutor.MODE_PROCESS: + if self.parallel_mode != ParallelDataExecutor.MODE_PROCESS and n_workers is not None: warnings.warn("ParallelDataExecutor: n_workers ({}) would be ignored for parallel_mode={}".format(n_workers, parallel_mode)) diff --git a/src/pyace/radial.py b/src/pyace/radial.py index c4cadb3..08ed7d8 100644 --- a/src/pyace/radial.py +++ b/src/pyace/radial.py @@ -17,7 +17,7 @@ def integrate(xs, table): frs = np.abs(table) sum_frs = np.sum(frs, axis=(1, 2)) integrand = sum_frs * xs ** 2 - integral = np.trapz(integrand, x=xs) + integral = np.trapezoid(integrand, x=xs) return integral else: return 0 diff --git a/tests/test_ACEAtomicEnvironment.py b/tests/test_ACEAtomicEnvironment.py index 9e49300..0142907 100644 --- a/tests/test_ACEAtomicEnvironment.py +++ b/tests/test_ACEAtomicEnvironment.py @@ -271,7 +271,7 @@ def test_generate_tp_atoms(): from ase.calculators.singlepoint import SinglePointCalculator spcalc = SinglePointCalculator(atoms, energy=-100, forces=np.zeros((len(atoms), 3))) - atoms.set_calculator(spcalc) + atoms.calc = spcalc tp_atoms = generate_tp_atoms(atoms, cutoff=r_cut) assert isinstance(tp_atoms, dict) diff --git a/tests/test_PyACECalculator.py b/tests/test_PyACECalculator.py index 2f71236..7cefe7a 100644 --- a/tests/test_PyACECalculator.py +++ b/tests/test_PyACECalculator.py @@ -43,7 +43,7 @@ def test_setup(): a = create_dimer(1) print(a) calc = PyACECalculator(basis_set=basisConfiguration) - a.set_calculator(calc) + a.calc = calc e1 = (a.get_potential_energy()) f1 = a.get_forces() print(e1) @@ -54,7 +54,7 @@ def test_load_YAML(): a = create_dimer(1) print(a) calc = PyACECalculator(basis_set="tests/Al.pbe.13.2.yaml") - a.set_calculator(calc) + a.calc = calc e1 = (a.get_potential_energy()) f1 = a.get_forces() print(e1) @@ -65,7 +65,7 @@ def test_load_ace(): a = create_dimer(1) print(a) calc = PyACECalculator(basis_set="tests/Al.pbe.rhocore.ace") - a.set_calculator(calc) + a.calc = calc e1 = (a.get_potential_energy()) f1 = a.get_forces() print(e1) @@ -82,7 +82,7 @@ def test_load_ace_recursive(): a = create_dimer(1) print(a) calc = PyACECalculator(basis_set="tests/Al.pbe.rhocore.ace", recursive_evaluator=True, recursive=True) - a.set_calculator(calc) + a.calc = calc e1 = (a.get_potential_energy()) f1 = a.get_forces() print(e1) @@ -99,7 +99,7 @@ def test_dimer_r1_energy_forces(): a = create_dimer(1) print(a) calc = PyACECalculator(basis_set="tests/Al-r1l0.yaml") - a.set_calculator(calc) + a.calc = calc energy = a.get_potential_energy() forces = a.get_forces() print(energy) @@ -113,7 +113,7 @@ def test_trimer_r234_energy_forces(): a = create_trimer(1) print(a) calculator = PyACECalculator(basis_set="tests/Al-r234.yaml") - a.set_calculator(calculator) + a.calc = calculator energy = a.get_potential_energy() forces = a.get_forces() print(energy) @@ -128,7 +128,7 @@ def test_load_YAML_pbc_symmetry_cubic(): print("a=", a) calc = PyACECalculator(basis_set="tests/Al.pbe.13.2.yaml") calc.cutoff = np.sqrt(2) * 1 - a.set_calculator(calc) + a.calc = calc e1 = a.get_potential_energy() f1 = a.get_forces() print("ae=", calc.ae) @@ -146,7 +146,7 @@ def test_load_YAML_pbc_symmetry_fcc_supercell(): a = bulk("Al", "fcc", cubic=True) * (1, 2, 3) print("a=", a) calc = PyACECalculator(basis_set="tests/Al.pbe.13.2.yaml") - a.set_calculator(calc) + a.calc = calc e1 = a.get_potential_energy() f1 = a.get_forces() print("ae=", calc.ae) @@ -165,7 +165,7 @@ def test_non_supported_element(): print("a=", a) calc = PyACECalculator(basis_set="tests/Al.pbe.13.2.yaml") with pytest.raises(ValueError): - a.set_calculator(calc) + a.calc = calc e1 = a.get_potential_energy() @@ -173,7 +173,7 @@ def test_fcc_stress(): a = bulk("Al", "fcc", a=4.03, cubic=True) * (1, 1, 1) print("a=", a) calc = PyACECalculator(basis_set="tests/Al.pbe.in-rank1.ace") - a.set_calculator(calc) + a.calc = calc e1 = a.get_potential_energy() f1 = a.get_forces() s1 = a.get_stress() @@ -188,12 +188,12 @@ def test_fcc_stress(): def test_relaxation(): - from ase.constraints import UnitCellFilter + from ase.filters import UnitCellFilter from ase.optimize import QuasiNewton calc = PyACECalculator(basis_set="tests/Al.pbe.rhocore-v2.ace") fcc = bulk("Al", cubic=True) - fcc.set_calculator(calc) + fcc.calc = calc print("Atoms before = ", fcc) e0 = fcc.get_potential_energy() f0 = fcc.get_forces() @@ -225,7 +225,7 @@ def test_PyACEEnsembleCalculator(): basis_sets = ["tests/Al.pbe.13.2.yaml", "tests/Al.pbe.rhocore.ace"] calc = PyACEEnsembleCalculator(basis_set=basis_sets) a = create_dimer(1) - a.set_calculator(calc) + a.calc = calc e1 = (a.get_potential_energy()) f1 = a.get_forces() print(e1) @@ -236,8 +236,11 @@ def test_ZBL_analytical_derivative(): calc = PyACECalculator("tests/ZBL_rep.yaml") def check(at, msg): - at.set_calculator(calc) - num_forces = calc.calculate_numerical_forces(at, d=1e-6) + at.calc = calc + import warnings + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + num_forces = calc.calculate_numerical_forces(at, d=1e-6) an_forces = at.get_forces() print(f"{num_forces=}, {an_forces=}") assert np.allclose(num_forces, an_forces), msg+f": {num_forces=}, {an_forces=}" diff --git a/tests/test_calculator.py b/tests/test_calculator.py index 2670403..e063c6d 100644 --- a/tests/test_calculator.py +++ b/tests/test_calculator.py @@ -203,7 +203,7 @@ def test_ASE_calculator_Voigt_stress_order(): new_cell = np.dot(cell, deform_matrix) test_Al.set_cell(new_cell, scale_atoms=True) - test_Al.set_calculator(calculator) + test_Al.calc = calculator stresses = test_Al.get_stress() print("stresses=", stresses) stresses_ref = [1.39710511e-01, 1.53093581e-01, 1.39710511e-01, -9.67787622e-05, @@ -219,7 +219,7 @@ def test_ASE_calculator_multispecies_AlNi(potname): chemsymb[0] = "Ni" test_Al.set_chemical_symbols(chemsymb) - test_Al.set_calculator(calculator) + test_Al.calc = calculator e = test_Al.get_potential_energy() ee = test_Al.get_potential_energies() @@ -254,7 +254,7 @@ def test_Ni_2dens_FexpShiftedScaled_manual_e_consistency(): print("f2.coeffs=", f2.coeffs) trimer = Atoms('Ni3', positions=[[1, 0, 0], [0, 1, 0], [0, 0, 1]], pbc=0) - trimer.set_calculator(calc) + trimer.calc = calc e = trimer.get_potential_energy() proj = calc.projections @@ -362,7 +362,7 @@ def test_PyACEEnsembleCalculator(): fcc_at = bulk("Al", "fcc", a=4.05, cubic=True) fnames = ["tests/Al.pbe.rhocore.ace", "tests/Al.pbe.rhocore.ace"] calc = PyACEEnsembleCalculator(fnames) - fcc_at.set_calculator(calc) + fcc_at.calc = calc e = fcc_at.get_potential_energy() print("e=", e) @@ -393,7 +393,7 @@ def test_PyACECalculator_active_set(): asecalc = PyACECalculator("tests/DFT10B-AgCu.yaml") asecalc.set_active_set("tests/DFT10B-AgCu.asi") - atoms.set_calculator(asecalc) + atoms.calc = asecalc energy = atoms.get_potential_energy() gamma = asecalc.results["gamma"] print("energy = ", energy) diff --git a/tests/test_generalfit.py b/tests/test_generalfit.py index d327cb8..adb3619 100644 --- a/tests/test_generalfit.py +++ b/tests/test_generalfit.py @@ -129,7 +129,8 @@ def compare_coefficients(all_coeffs, all_coeffs_ref, abs_threshold=2e-7, rel_thr print("abs diff max = ", np.max(dcoeff)) print("abs diff norm = ", np.linalg.norm(dcoeff)) - rel_err = 2 * dcoeff / (all_coeffs + all_coeffs_ref) + with np.errstate(divide='ignore', invalid='ignore'): + rel_err = 2 * dcoeff / (all_coeffs + all_coeffs_ref) print("rel_err = ", rel_err) rel_norm = np.linalg.norm(rel_err) print("rel_err max= ", np.max(rel_err)) diff --git a/tests/test_linearfit.py b/tests/test_linearfit.py index 1c81298..8d21d33 100644 --- a/tests/test_linearfit.py +++ b/tests/test_linearfit.py @@ -113,7 +113,7 @@ def test_LinearACEFit(): e_pred, f_pred = linear_fit.predict(ds, reshape_forces=True) # take first ase_atoms at = df.iloc[0]["ase_atoms"].copy() - at.set_calculator(calc) + at.calc = calc at.get_potential_energy() assert np.allclose(at.get_potential_energy() / len(at), e_pred[0]) assert np.allclose(at.get_forces(), f_pred[:len(at)]) From b3502139c8c9b507c2cf2b9a8831895eec6b47f3 Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Mon, 23 Feb 2026 14:58:47 +0100 Subject: [PATCH 06/30] replace `.set_calculator` with `.calc` --- src/pyace/preparedata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pyace/preparedata.py b/src/pyace/preparedata.py index 68e94c5..129d04f 100644 --- a/src/pyace/preparedata.py +++ b/src/pyace/preparedata.py @@ -58,7 +58,7 @@ def attach_single_point_calculator(row): energy = row["energy_corrected"] forces = row["forces"] calc = SinglePointCalculator(atoms, energy=energy, forces=forces) - atoms.set_calculator(calc) + atoms.calc=calc return atoms From d0f7a0bd843d899b31084ca1b4a3d502e4afa727 Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Mon, 23 Feb 2026 15:02:47 +0100 Subject: [PATCH 07/30] adjust numpy<2.2.0 with tensorflow 2.19.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 84c6f39..7424e41 100644 --- a/setup.py +++ b/setup.py @@ -193,7 +193,7 @@ def build_extension(self, ext: CMakeExtension) -> None: build_ext=CMakeBuild)), zip_safe=False, url='https://github.com/ICAMS/python-ace', - install_requires=['numpy>=2.0.0', + install_requires=['numpy<2.2.0', 'ase', 'pandas', 'ruamel.yaml', From df3350d6d904dc53815f85c268f4c4e2caf9f5cb Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Fri, 4 Sep 2026 11:09:32 +0200 Subject: [PATCH 08/30] build: restore debuggable Debug builds CMAKE_CXX_FLAGS_DEBUG was set to "-Ofast -DNDEBUG", making a Debug build byte-for-byte equivalent to Release: no -g, no symbols, fully inlined. There was no way to get a debuggable binary without editing this file. Set it to "-Og -g" instead. Verified via generated flags.make that Release (the default, forced when CMAKE_BUILD_TYPE is unset) still compiles with "-Ofast -DNDEBUG -flto" and is unaffected; only DEBUG=1 / -DCMAKE_BUILD_TYPE=Debug sees the new flags. Also dropped the now-redundant commented-out '#set(CMAKE_CXX_FLAGS_DEBUG "-g")' directly above, which contradicted the active setting. --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e2d5b68..cf7ca9b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -139,9 +139,9 @@ set(GRACE_FS_SOURCES # C++ FLAGS #--------------------------------------------------------- #set(CMAKE_CXX_FLAGS "-Wall -Wextra") -#set(CMAKE_CXX_FLAGS_DEBUG "-g") set(CMAKE_CXX_FLAGS_RELEASE "-Ofast -DNDEBUG") -set(CMAKE_CXX_FLAGS_DEBUG "-Ofast -DNDEBUG") +# keep Debug builds debuggable (only used with -DCMAKE_BUILD_TYPE=Debug / DEBUG=1) +set(CMAKE_CXX_FLAGS_DEBUG "-Og -g") #Finally create the package #-------------------------------------------------------- From 70a620f113a459cd2b7173aa7c8f47bcb054046a Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Fri, 4 Sep 2026 12:10:22 +0200 Subject: [PATCH 09/30] build: compile shared ACE sources once via OBJECT libraries Five translation units were listed in four different module targets each (ace_radial, ace_spherical_cart, ace_abstract_basis, ace_clebsch_gordan, ace_couplings), and CMake never shares object files between targets, so each was compiled once per consuming module. Group them into OBJECT libraries keyed on the *exact* set of modules that already used each source, so every module links precisely the objects it linked before. The grouping falls out cleanly into five libraries; no module gains or loses an object. The object libraries inherit the same compile settings pybind11_add_module applies to the modules -- PIC, hidden visibility (needed: some of these are pybind11 binding units) and the same LTO selection via pybind11::lto -- which was verified against the generated build.ninja: the flags for a shared TU and a module TU are identical. Project object files: 59 -> 30. --- CMakeLists.txt | 131 ++++++++++++++++++++++++++++++------------------- 1 file changed, 80 insertions(+), 51 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cf7ca9b..30011ed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -69,71 +69,103 @@ add_subdirectory(lib/ace) #Now add sources #-------------------------------------------------------- -set(SOURCES_SHARMONICS "${ACE_SRC}/ace_couplings.cpp" - "${ACE_SRC}/ace_spherical_polar.cpp" - "${ACE_SRC}/ace_clebsch_gordan.cpp" - "${SOURCE_DIR}/ace/ace_spherical_polar_binding.cpp" - ) -set(SOURCES_COUPLING "${ACE_SRC}/ace_couplings.cpp" +# Translation units shared by more than one module live in OBJECT libraries so +# they are compiled once instead of once per consuming module. The grouping is +# by the *exact* set of modules that already used each source, so every module +# still links precisely the objects it linked before. +# +# Compile settings match what pybind11_add_module applies to the modules: PIC, +# hidden visibility (required -- some of these are pybind11 binding units), and +# the same LTO selection, so each object is byte-identical to before. +function(ace_add_object_library name) + add_library(${name} OBJECT ${ARGN}) + target_link_libraries(${name} PRIVATE pybind11::module) + if (TARGET pybind11::lto) + target_link_libraries(${name} PRIVATE pybind11::lto) + endif () + set_target_properties(${name} PROPERTIES + POSITION_INDEPENDENT_CODE ON + CXX_VISIBILITY_PRESET hidden) + if (CMAKE_COMPILER_IS_GNUCC) + target_compile_options(${name} PRIVATE -Wunused-variable -Wvla -Wno-effc++ -Werror=vla) + endif () +endfunction() + +# used by: sharmonics, coupling, basis, evaluator +ace_add_object_library(ace_coupling_obj "${ACE_SRC}/ace_clebsch_gordan.cpp" - "${SOURCE_DIR}/ace/ace_coupling_binding.cpp") + "${ACE_SRC}/ace_couplings.cpp") -set(SOURCES_BASIS "${ACE_EVALUATOR_SRC}/ace_c_basis.cpp" - "${ACE_EVALUATOR_SRC}/ace_radial.cpp" - "${ACE_EVALUATOR_SRC}/ace_spherical_cart.cpp" +# used by: basis, evaluator, calculator, grace_fs +ace_add_object_library(ace_radial_obj "${ACE_EVALUATOR_SRC}/ace_abstract_basis.cpp" + "${ACE_EVALUATOR_SRC}/ace_radial.cpp" + "${ACE_EVALUATOR_SRC}/ace_spherical_cart.cpp") + +# used by: catomicenvironment, evaluator, calculator +ace_add_object_library(ace_atoms_obj + "${ACE_EVALUATOR_EXTRA}/ace_atoms.cpp") + +# used by: evaluator, calculator +ace_add_object_library(ace_evaluator_obj + "${ACE_EVALUATOR_SRC}/ace_evaluator.cpp") + +# used by: basis, evaluator +ace_add_object_library(ace_bbasis_obj + "${ACE_EVALUATOR_SRC}/ace_c_basis.cpp" "${ACE_EVALUATOR_SRC}/ace_flatten_basis.cpp" "${ACE_EVALUATOR_SRC}/ships_radial.cpp" "${ACE_SRC}/ace_b_basis.cpp" "${ACE_SRC}/ace_b_basisfunction.cpp" - "${ACE_SRC}/ace_clebsch_gordan.cpp" - "${ACE_SRC}/ace_couplings.cpp" "${ACE_SRC}/ace_yaml_input.cpp" - "${SOURCE_DIR}/ace-evaluator/ace_c_basis_binding.cpp" "${SOURCE_DIR}/ace-evaluator/ace_bbasis_spec_helper.cpp" - "${SOURCE_DIR}/ace/ace_radial_helper.cpp" - "${SOURCE_DIR}/ace/ace_c_basisfunction_helper.cpp" + "${SOURCE_DIR}/ace-evaluator/ace_c_basis_binding.cpp" "${SOURCE_DIR}/ace/ace_c_basis_helper.cpp" - ) - -set(SOURCES_EVALUATOR "${ACE_EVALUATOR_SRC}/ace_c_basis.cpp" - "${ACE_EVALUATOR_SRC}/ace_abstract_basis.cpp" - "${ACE_EVALUATOR_SRC}/ace_flatten_basis.cpp" - "${ACE_EVALUATOR_SRC}/ace_evaluator.cpp" - "${ACE_EVALUATOR_SRC}/ace_recursive.cpp" - "${ACE_EVALUATOR_EXTRA}/ace_atoms.cpp" - "${ACE_EVALUATOR_SRC}/ace_radial.cpp" - "${ACE_EVALUATOR_SRC}/ace_spherical_cart.cpp" - "${ACE_SRC}/ace_b_evaluator.cpp" - "${ACE_SRC}/ace_b_basis.cpp" - "${ACE_SRC}/ace_clebsch_gordan.cpp" - "${ACE_SRC}/ace_yaml_input.cpp" - "${ACE_SRC}/ace_couplings.cpp" - "${SOURCE_DIR}/ace-evaluator/ace_evaluator_binding.cpp" - ) - -set(SOURCES_CATOMICENVIRONMENT "${ACE_EVALUATOR_EXTRA}/ace_atoms.cpp" - "${SOURCE_DIR}/utils/ace_atoms_binding.cpp") - + "${SOURCE_DIR}/ace/ace_c_basisfunction_helper.cpp" + "${SOURCE_DIR}/ace/ace_radial_helper.cpp") + +# Per-module sources: only the units unique to that module. +set(SOURCES_SHARMONICS + "${ACE_SRC}/ace_spherical_polar.cpp" + "${SOURCE_DIR}/ace/ace_spherical_polar_binding.cpp" + $) + +set(SOURCES_COUPLING + "${SOURCE_DIR}/ace/ace_coupling_binding.cpp" + $) + +set(SOURCES_CATOMICENVIRONMENT + "${SOURCE_DIR}/utils/ace_atoms_binding.cpp" + $) + +set(SOURCES_BASIS + $ + $ + $) + +set(SOURCES_EVALUATOR + "${ACE_EVALUATOR_SRC}/ace_recursive.cpp" + "${ACE_SRC}/ace_b_evaluator.cpp" + "${SOURCE_DIR}/ace-evaluator/ace_evaluator_binding.cpp" + $ + $ + $ + $ + $) set(SOURCES_CALCULATOR - "${ACE_EVALUATOR_SRC}/ace_radial.cpp" - "${ACE_EVALUATOR_SRC}/ace_abstract_basis.cpp" - "${ACE_EVALUATOR_EXTRA}/ace_atoms.cpp" - "${ACE_EVALUATOR_SRC}/ace_spherical_cart.cpp" "${ACE_EVALUATOR_EXTRA}/ace_calculator.cpp" - "${ACE_EVALUATOR_SRC}/ace_evaluator.cpp" - "${SOURCE_DIR}/utils/ace_calculator_binding.cpp") + "${SOURCE_DIR}/utils/ace_calculator_binding.cpp" + $ + $ + $) set(GRACE_FS_SOURCES - "${ACE_EVALUATOR_SRC}/ace_abstract_basis.cpp" - "${ACE_EVALUATOR_SRC}/ace_radial.cpp" - "${ACE_EVALUATOR_SRC}/ace_spherical_cart.cpp" - "${SOURCE_DIR}/ace/grace_fs_binding.cpp" "${ACE_SRC}/grace_fs_evaluator.cpp" "${ACE_EXTRA_SRC}/grace_fs_calculator.cpp" -) + "${SOURCE_DIR}/ace/grace_fs_binding.cpp" + $) # C++ FLAGS @@ -168,15 +200,13 @@ if (CMAKE_COMPILER_IS_GNUCC) target_compile_options(basis PRIVATE -Wunused-variable -Wvla -Wno-effc++ -Werror=vla) endif () -pybind11_add_module(evaluator ${SOURCES_EVALUATOR} ${SOURCES_BASIS} ) +pybind11_add_module(evaluator ${SOURCES_EVALUATOR} ) target_link_libraries(evaluator PRIVATE yaml-cpp-pace) target_link_libraries(evaluator PRIVATE cnpy) if (CMAKE_COMPILER_IS_GNUCC) target_compile_options(evaluator PRIVATE -Wunused-variable -Wvla -Wno-effc++ -Werror=vla) endif () - - pybind11_add_module(grace_fs ${GRACE_FS_SOURCES}) target_link_libraries(grace_fs PRIVATE yaml-cpp-pace) target_link_libraries(grace_fs PRIVATE cnpy) @@ -188,4 +218,3 @@ pybind11_add_module(calculator ${SOURCES_CALCULATOR} ) if (CMAKE_COMPILER_IS_GNUCC) target_compile_options(calculator PRIVATE -Wunused-variable -Wvla -Wno-effc++ -Werror=vla) endif () - From b64e60ea16b94a7f32158d3b27cfe1d73f7fb92f Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Fri, 4 Sep 2026 12:10:44 +0200 Subject: [PATCH 10/30] build: single CMake tree, safer job count, ccache; vendor fp64 maxvol Two coupled changes: reworking build_ext to drive one CMake tree is what makes it possible to build maxvol as an ordinary extension alongside the CMake ones. Build ----- setuptools calls build_extension() once per Extension, and the old code gave each its own build directory -- seven CMake configures and seven independent builds per `pip install .`, recompiling yaml-cpp four times and cnpy three. Override build_extensions() instead: one configure, one build of the default target (which already covers all seven modules, so no CMake 3.15 multi-target syntax is needed). Non-CMake extensions still go through setuptools. Job count: os.cpu_count() reports host CPUs, so a container pinned to two cores on a large host got -j19. Use the affinity mask, keep both cores on a 2-core box rather than reserving one, cap by available memory (cgroup v1/v2 aware), and stop overriding an explicit `build_ext -j`. Use ccache/sccache as compiler launcher when present; PYACE_NO_COMPILER_CACHE opts out. Add pyproject.toml declaring the build deps -- ninja was silently absent, so the build had been falling back to make. Measured in the tp2 env (20 cores, full test suite passing in all cases): baseline 126 s wall / 333 s CPU / 178 objects / 7 configures after 53 s wall / 181 s CPU / 60 objects / 1 configure after + ccache 32 s wall / 71 s CPU, peak RSS 780 MB -> 103 MB 2 cores (taskset): 182 s -> ~156 s wall, 255 s -> 143 s CPU maxvol ------ maxvolpy was installed by a custom `install` cmdclass that pip never runs when it builds a wheel, so `pip install .` produced a pyace whose active-learning entry points could not import; CI worked around it with an explicit `pip install ./lib/maxvolpy`. It also compiled with -march=native, which bakes the build machine's ISA into the binary, and -ffast-math, which is a poor fit for a pivoting algorithm whose comparisons decide which rows get selected. Vendor just the float64 square maxvol into src/pyace/maxvol (716 -> 127 lines of Cython; the float32/complex variants and the rect_maxvol family are not used by pyace) and build it as a normal cythonised extension. Keep a pure numpy/scipy py_maxvol fallback so a missing or ABI-mismatched binary degrades to slower rather than to ImportError. Verified the compiled kernel and the fallback select identical rows with max|C| agreeing to ~1e-15, on 10k x 200, 50k x 300 and 100k x 300; the same figures reproduce the original maxvolpy results. Fallback exercised by blocking the extension import. Two fixes carried in the reduced kernel: the pivot-interchange loops used prange, which is a data race on order-dependent swaps and was only safe because nothing passed -fopenmp; and the scratch buffers now free in a finally, so an error from dgetrf no longer leaks them. numpy>=2.0 is now pinned at both build and run time, since the extension compiles against the numpy C-API and the build-time numpy sets the runtime floor. scipy becomes an explicit dependency (cython_blas/cython_lapack). MIT notice for the vendored code ships in the wheel. --- .github/workflows/test.yml | 10 - .gitignore | 4 +- pyproject.toml | 20 ++ setup.py | 274 +++++++++++++++++--------- src/pyace/activelearning.py | 2 +- src/pyace/maxvol/LICENSE.maxvolpy.txt | 20 ++ src/pyace/maxvol/__init__.py | 119 +++++++++++ src/pyace/maxvol/_maxvol.pyx | 127 ++++++++++++ 8 files changed, 470 insertions(+), 106 deletions(-) create mode 100644 pyproject.toml create mode 100644 src/pyace/maxvol/LICENSE.maxvolpy.txt create mode 100644 src/pyace/maxvol/__init__.py create mode 100644 src/pyace/maxvol/_maxvol.pyx diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e3d4127..512f43a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,16 +42,6 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi #python setup.py install pip install . - - name: Install maxvolpy - # `pip install .` builds a wheel and never runs setup.py's custom - # InstallMaxVolPyLocalPackage(install) hook, so lib/maxvolpy is not - # installed and the active-learning tests fail to import. Build it - # explicitly; --no-build-isolation is required because - # lib/maxvolpy/setup.py imports Cython and numpy at module level - # without declaring them in a pyproject.toml build-system. - run: | - pip install Cython scipy - pip install --no-build-isolation ./lib/maxvolpy - name: Test with python-ace with tensorpotential run: | python -m pytest tests/ --runtensorpot diff --git a/.gitignore b/.gitignore index c3c9504..6a79ed1 100644 --- a/.gitignore +++ b/.gitignore @@ -143,4 +143,6 @@ cython_debug/ media static -cmake-build-*/ \ No newline at end of file +cmake-build-*/ +# generated by cython +src/pyace/maxvol/_maxvol.c diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..0d0d6be --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,20 @@ +[build-system] +# Declaring the build dependencies means `pip install .` provisions them itself +# instead of silently degrading: without ninja the CMake build falls back to +# make, which schedules this project's seven modules much less effectively, and +# without Cython the maxvol extension is skipped entirely. +# +# numpy>=2.0 is a *build* pin, not a preference: pyace.maxvol._maxvol is a +# Cython extension against the numpy C-API, so the numpy it is compiled against +# sets the minimum numpy it will import under at runtime. Keep this consistent +# with install_requires in setup.py. +requires = [ + "setuptools>=64", + "wheel", + "cmake>=3.15", + "ninja; platform_system!='Windows'", + "Cython>=0.29.32", + "numpy>=2.0", + "scipy", +] +build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py index 7424e41..8397cdd 100644 --- a/setup.py +++ b/setup.py @@ -1,13 +1,13 @@ +from __future__ import annotations + import os import re +import shutil import subprocess import sys from pathlib import Path -import platform -from distutils.version import LooseVersion from setuptools import Extension, setup, find_packages from setuptools.command.build_ext import build_ext -from setuptools.command.install import install import versioneer @@ -15,23 +15,6 @@ readme = readme_file.read() -class InstallMaxVolPyLocalPackage(install): - def run(self): - install.run(self) - cmd = "cd lib/maxvolpy; python setup.py install; cd ../.." - if platform.system() != "Windows": - cmd = "pip install Cython; " + cmd - returncode = subprocess.call( - cmd, shell=True - ) - if returncode != 0: - print("=" * 40) - print("=" * 16, "WARNING", "=" * 17) - print("=" * 40) - print("Installation of `lib/maxvolpy` return {} code!".format(returncode)) - print("Active learning/selection of active set will not work!") - - # Convert distutils Windows platform specifiers to CMake -A arguments PLAT_TO_CMAKE = { "win32": "Win32", @@ -45,60 +28,138 @@ def run(self): # The name must be the _single_ output extension from the CMake build. # If you need multiple extensions, see scikit-build. class CMakeExtension(Extension): - def __init__(self, name: str, target=None, sourcedir: str = "") -> None: + def __init__(self, name: str, sourcedir: str = "") -> None: super().__init__(name, sources=[]) self.sourcedir = os.fspath(Path(sourcedir).resolve()) - self.target = target + + +def _cgroup_memory_limit() -> int | None: + """Memory ceiling imposed by a container, or None if unlimited.""" + for path, parse in ( + ("/sys/fs/cgroup/memory.max", lambda s: None if s.strip() == "max" else int(s)), + ("/sys/fs/cgroup/memory/memory.limit_in_bytes", int), + ): + try: + value = parse(Path(path).read_text()) + except (OSError, ValueError): + continue + # cgroup v1 reports a huge sentinel rather than omitting the limit + if value and value < (1 << 62): + return value + return None + + +def _available_memory() -> int | None: + limit = _cgroup_memory_limit() + try: + for line in Path("/proc/meminfo").read_text().splitlines(): + if line.startswith("MemAvailable:"): + available = int(line.split()[1]) * 1024 + return min(available, limit) if limit else available + except (OSError, ValueError, IndexError): + pass + return limit + + +def default_parallel_jobs() -> int: + """ + Choose a compile job count that is safe on small machines. + + os.cpu_count() reports the *host* CPU count, so inside a container pinned to + two cores on a large host it returns the host's count and we end up spawning + dozens of compilers -- which thrashes or gets OOM-killed. The affinity mask + reflects taskset and cpuset limits, so prefer it. Cap by available memory as + well: these translation units pull in pybind11 and are linked with -flto, so + budget ~2 GB per concurrent job. + """ + try: + cpus = len(os.sched_getaffinity(0)) # Linux only; honours taskset/cpuset + except AttributeError: + cpus = os.cpu_count() or 1 + + # Reserve a core for the rest of the system, but not when that would halve + # throughput: on a 2-core runner we want both cores. + jobs = cpus if cpus <= 2 else cpus - 1 + + memory = _available_memory() + if memory: + jobs = max(1, min(jobs, memory // (2 * 1024 ** 3))) + return jobs + + +def compiler_launcher() -> str | None: + """Path to ccache/sccache if one is installed and not disabled.""" + if os.environ.get("PYACE_NO_COMPILER_CACHE"): + return None + for name in ("ccache", "sccache"): + path = shutil.which(name) + if path: + return path + return None class CMakeBuild(build_ext): + """ + Configure and build every CMake extension in a single build tree. + + setuptools calls build_extension() once per Extension. Doing the CMake work + there gave each module its own build directory, so `pip install .` ran seven + configures and seven independent builds -- recompiling yaml-cpp four times, + cnpy three times, and the shared ACE sources once per consuming module. + Building all of them from one tree compiles each object exactly once and + lets the generator schedule every module's work in parallel. + """ + + def build_extensions(self) -> None: + cmake_extensions = [e for e in self.extensions if isinstance(e, CMakeExtension)] + other_extensions = [e for e in self.extensions if not isinstance(e, CMakeExtension)] + if not cmake_extensions: + super().build_extensions() + return - def build_extension(self, ext: CMakeExtension) -> None: try: - out = subprocess.check_output(['cmake', '--version']) + subprocess.check_output(["cmake", "--version"]) except OSError: + raise RuntimeError("CMake must be installed to build the extensions") + + # Every module is placed in the same package, so one output directory + # serves them all. Assert it rather than assume it. + outdirs = { + Path.cwd().joinpath(self.get_ext_fullpath(e.name)).parent.resolve() + for e in cmake_extensions + } + if len(outdirs) != 1: raise RuntimeError( - "CMake must be installed to build the extensions") - self.parallel = os.cpu_count() - 1 - if self.parallel < 1: - self.parallel = 1 - # Must be in this form due to bug in .resolve() only fixed in Python 3.10+ - ext_fullpath = Path.cwd() / self.get_ext_fullpath(ext.name) - extdir = ext_fullpath.parent.resolve() - - # Using this requires trailing slash for auto-detection & inclusion of - # auxiliary "native" libs + "CMake extensions must share one output directory, got: " + + ", ".join(sorted(str(d) for d in outdirs)) + ) + extdir = outdirs.pop() debug = int(os.environ.get("DEBUG", 0)) if self.debug is None else self.debug cfg = "Debug" if debug else "Release" - # CMake lets you override the generator - we need to check this. - # Can be set with Conda-Build, for example. cmake_generator = os.environ.get("CMAKE_GENERATOR", "") - # Set Python_EXECUTABLE instead if you use PYBIND11_FINDPYTHON - # EXAMPLE_VERSION_INFO shows you how to pass a value into the C++ code - # from Python. cmake_args = [ f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}{os.sep}", f"-DPYTHON_EXECUTABLE={sys.executable}", f"-DCMAKE_BUILD_TYPE={cfg}", # not used on MSVC, but no harm ] + + launcher = compiler_launcher() + if launcher: + cmake_args += [ + f"-DCMAKE_C_COMPILER_LAUNCHER={launcher}", + f"-DCMAKE_CXX_COMPILER_LAUNCHER={launcher}", + ] + build_args = [] - # Adding CMake arguments set as environment variable - # (needed e.g. to build for ARM OSx on conda-forge) if "CMAKE_ARGS" in os.environ: cmake_args += [item for item in os.environ["CMAKE_ARGS"].split(" ") if item] - # In this example, we pass in the version to C++. You might not need to. - # cmake_args += [f"-DEXAMPLE_VERSION_INFO={self.distribution.get_version()}"] - if self.compiler.compiler_type != "msvc": - # Using Ninja-build since it a) is available as a wheel and b) - # multithreads automatically. MSVC would require all variables be - # exported for Ninja to pick it up, which is a little tricky to do. - # Users can override the generator with CMAKE_GENERATOR in CMake - # 3.15+. + # Ninja schedules across all modules far better than Make here, and + # is available as a wheel. Users can override with CMAKE_GENERATOR. if not cmake_generator or cmake_generator == "Ninja": try: import ninja @@ -110,56 +171,77 @@ def build_extension(self, ext: CMakeExtension) -> None: ] except ImportError: pass - else: - # Single config generators are handled "normally" single_config = any(x in cmake_generator for x in {"NMake", "Ninja"}) - - # CMake allows an arch-in-generator style for backward compatibility contains_arch = any(x in cmake_generator for x in {"ARM", "Win64"}) - # Specify the arch if using MSVC generator, but only if it doesn't - # contain a backward-compatibility arch spec already in the - # generator name. if not single_config and not contains_arch: cmake_args += ["-A", PLAT_TO_CMAKE[self.plat_name]] - # Multi-config generators have a different way to specify configs if not single_config: - cmake_args += [ - f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}" - ] + cmake_args += [f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}"] build_args += ["--config", cfg] - if ext.target is not None: - build_args += ["--target", ext.target] - if sys.platform.startswith("darwin"): - # Cross-compile support for macOS - respect ARCHFLAGS if set archs = re.findall(r"-arch (\S+)", os.environ.get("ARCHFLAGS", "")) if archs: cmake_args += ["-DCMAKE_OSX_ARCHITECTURES={}".format(";".join(archs))] - # Set CMAKE_BUILD_PARALLEL_LEVEL to control the parallel build level - # across all generators. + # CMAKE_BUILD_PARALLEL_LEVEL, then an explicit `build_ext -j`, then our + # own estimate. Never silently override what the caller asked for. if "CMAKE_BUILD_PARALLEL_LEVEL" not in os.environ: - # self.parallel is a Python 3 only way to set parallel jobs by hand - # using -j in the build_ext call, not supported by pip or PyPA-build. - if hasattr(self, "parallel") and self.parallel: - # CMake 3.12+ only. - build_args += [f"-j{self.parallel}"] - - build_temp = Path(self.build_temp) / ext.name - if not build_temp.exists(): - build_temp.mkdir(parents=True) - - subprocess.run( - ["cmake", ext.sourcedir, *cmake_args], cwd=build_temp, check=True - ) - args = ["cmake", "--build", ".", *build_args] - subprocess.run( - args, cwd=build_temp, check=True + if not getattr(self, "parallel", None): + self.parallel = default_parallel_jobs() + build_args += [f"-j{self.parallel}"] + + build_temp = Path(self.build_temp) + build_temp.mkdir(parents=True, exist_ok=True) + + sourcedir = cmake_extensions[0].sourcedir + subprocess.run(["cmake", sourcedir, *cmake_args], cwd=build_temp, check=True) + # No --target: the default target already covers every module, which + # keeps this working on CMake older than the 3.15 multi-target syntax. + subprocess.run(["cmake", "--build", ".", *build_args], cwd=build_temp, check=True) + + # Anything that is not a CMakeExtension (e.g. a cythonised extension) + # is still built the ordinary setuptools way. + if other_extensions: + saved, self.extensions = self.extensions, other_extensions + try: + super().build_extensions() + finally: + self.extensions = saved + + +def maxvol_extension(): + """ + The float64 maxvol kernel, built as an ordinary cythonised extension. + + It used to live in lib/maxvolpy and was installed by a custom `install` + command, which pip never runs when it builds a wheel -- so `pip install .` + silently produced a pyace whose active-learning entry points could not + import. Building it here makes it part of the wheel like any other module. + + Cython is resolved lazily so that a tree without it (e.g. `setup.py --help`) + still imports; the build itself declares Cython in pyproject.toml. + """ + try: + from Cython.Build import cythonize + import numpy + except ImportError: + return [] + return cythonize([ + Extension( + "pyace.maxvol._maxvol", + ["src/pyace/maxvol/_maxvol.pyx"], + include_dirs=[numpy.get_include()], + # No -march=native: it bakes in the build machine's ISA and + # SIGILLs on any older CPU, which makes wheels non-redistributable. + # No -ffast-math either -- this is a pivoting algorithm whose + # comparisons decide which rows get selected. + extra_compile_args=["-O3"] if os.name != "nt" else [], ) + ], language_level=3) # The information here can also be placed in setup.cfg - better separation of @@ -180,25 +262,26 @@ def build_extension(self, ext: CMakeExtension) -> None: package_dir={'': 'src'}, # add an extension module named 'python_cpp_example' to the package - ext_modules=[CMakeExtension('pyace/sharmonics', target='sharmonics'), - CMakeExtension('pyace/coupling', target='coupling'), - CMakeExtension('pyace/basis', target='basis'), - CMakeExtension('pyace/evaluator', target='evaluator'), - CMakeExtension('pyace/catomicenvironment', target='catomicenvironment'), - CMakeExtension('pyace/calculator', target='calculator'), - CMakeExtension('pyace/grace_fs', target='grace_fs'), + ext_modules=[CMakeExtension('pyace/sharmonics'), + CMakeExtension('pyace/coupling'), + CMakeExtension('pyace/basis'), + CMakeExtension('pyace/evaluator'), + CMakeExtension('pyace/catomicenvironment'), + CMakeExtension('pyace/calculator'), + CMakeExtension('pyace/grace_fs'), + *maxvol_extension(), ], # add custom build_ext command - cmdclass=versioneer.get_cmdclass(dict(install=InstallMaxVolPyLocalPackage, - build_ext=CMakeBuild)), + cmdclass=versioneer.get_cmdclass(dict(build_ext=CMakeBuild)), zip_safe=False, url='https://github.com/ICAMS/python-ace', - install_requires=['numpy<2.2.0', + install_requires=['numpy>=2.0,<2.2.0', 'ase', 'pandas', 'ruamel.yaml', 'psutil', - 'scikit-learn' + 'scikit-learn', + 'scipy' ], classifiers=[ 'Programming Language :: Python :: 3', @@ -206,7 +289,10 @@ def build_extension(self, ext: CMakeExtension) -> None: package_data={"pyace.data": [ "mus_ns_uni_to_rawlsLS_np_rank.pckl", "input_template.yaml" - ]}, + ], + # MIT notice must travel with the vendored maxvol kernel + "pyace.maxvol": ["LICENSE.maxvolpy.txt"], + }, scripts=["bin/pacemaker", "bin/pace_yaml2yace", "bin/pace_timing", "bin/pace_info", "bin/pace_activeset", "bin/pace_select", diff --git a/src/pyace/activelearning.py b/src/pyace/activelearning.py index 7355ff8..e2eb45f 100644 --- a/src/pyace/activelearning.py +++ b/src/pyace/activelearning.py @@ -7,7 +7,7 @@ import pandas as pd from ase import Atoms from ase.io.lammpsrun import read_lammps_dump_text -from maxvolpy.maxvol import maxvol +from pyace.maxvol import maxvol from pyace.asecalc import PyACECalculator, PyGRACEFSCalculator from pyace.atomicenvironment import aseatoms_to_atomicenvironment, ACEAtomicEnvironment diff --git a/src/pyace/maxvol/LICENSE.maxvolpy.txt b/src/pyace/maxvol/LICENSE.maxvolpy.txt new file mode 100644 index 0000000..e03217d --- /dev/null +++ b/src/pyace/maxvol/LICENSE.maxvolpy.txt @@ -0,0 +1,20 @@ +Copyright (c) 2015 +Alexander Mikhalev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/pyace/maxvol/__init__.py b/src/pyace/maxvol/__init__.py new file mode 100644 index 0000000..272527a --- /dev/null +++ b/src/pyace/maxvol/__init__.py @@ -0,0 +1,119 @@ +""" +Row selection by 1-volume maximisation (maxvol). + +Reduced from maxvolpy 0.3.8 by Alexander Mikhalev, MIT licence -- see +LICENSE.maxvolpy.txt next to this file. pyace only ever needs the float64 +square `maxvol`, so only that is kept. + +`maxvol` prefers the compiled `_maxvol.c_maxvol`. If that extension is missing +or fails to load, it falls back to `py_maxvol`, which selects exactly the same +rows via LAPACK/BLAS through scipy but is roughly 3-15x slower depending on +matrix size. +""" +import logging + +import numpy as np +from scipy.linalg import get_lapack_funcs, get_blas_funcs + +__all__ = ['maxvol', 'py_maxvol'] + +log = logging.getLogger(__name__) + + +def py_maxvol(A, tol=1.05, max_iters=100, top_k_index=-1, verbose=False): + """ + Pure numpy/scipy 1-volume maximisation. + + :param A: (N, r) float64 array with N > r + :param tol: stop once no coefficient exceeds this, must be >= 1 + :param max_iters: maximum number of row swaps + :param top_k_index: only consider the first `top_k_index` rows (-1 = all) + :return: (row_indices, coefficients), the same pair `c_maxvol` returns + """ + if tol < 1: + tol = 1.0 + N, r = A.shape + if N <= r: + return np.arange(N, dtype=np.int32), np.eye(N, dtype=A.dtype) + if top_k_index == -1 or top_k_index > N: + top_k_index = N + if top_k_index < r: + top_k_index = r + + B = np.copy(A[:top_k_index], order='F') + C = np.copy(A.T, order='F') + H, ipiv, info = get_lapack_funcs('getrf', [B])(B, overwrite_a=1) + if info < 0: + raise ValueError("argument {} of getrf had an illegal value".format(-info)) + if info > 0: + raise ValueError("Input matrix must not be singular") + + index = np.arange(N, dtype=np.int32) + for i in range(r): + index[i], index[ipiv[i]] = index[ipiv[i]], index[i] + + # solve A = CH with H in LU form + B = H[:r] + trtrs = get_lapack_funcs('trtrs', [B]) + trtrs(B, C, trans=1, lower=0, unitdiag=0, overwrite_b=1) + trtrs(B, C, trans=1, lower=1, unitdiag=1, overwrite_b=1) + + # C is stored transposed, shape (r, N) + view = C[:, :top_k_index] + # reused every iteration -- np.abs(view) would otherwise allocate an + # N-by-r temporary per swap, which dominates the runtime for large N + abs_view = np.empty_like(view) + np.abs(view, out=abs_view) + i, j = divmod(abs_view.argmax(), top_k_index) + + ger = get_blas_funcs('ger', [C]) + iters = 0 + while abs(C[i, j]) > tol and iters < max_iters: + if verbose and iters % 10 == 0: + log.info("Iter %d/%d: abs_max = %.3f (tol = %.3f)", + iters, max_iters, abs(C[i, j]), tol) + index[i] = j + tmp_row = C[i].copy() + tmp_column = C[:, j].copy() + tmp_column[i] -= 1. + ger(-1. / C[i, j], tmp_column, tmp_row, a=C, overwrite_a=1) + iters += 1 + np.abs(view, out=abs_view) + i, j = divmod(abs_view.argmax(), top_k_index) + return index[:r].copy(), C.T + + +try: + from ._maxvol import c_maxvol + + _maxvol_func = c_maxvol + __all__.append('c_maxvol') +except ImportError as e: # pragma: no cover - depends on how pyace was built + _maxvol_func = None + _import_error = e + + +def maxvol(A, tol=1.05, max_iters=100, top_k_index=-1, verbose=False): + """ + Find `r` rows of the (N, r) array `A` whose submatrix has near-maximal + absolute determinant. + + :param A: (N, r) float64 array + :param tol: stop once no coefficient exceeds this, must be >= 1 + :param max_iters: maximum number of row swaps + :param top_k_index: only consider the first `top_k_index` rows (-1 = all) + :param verbose: log progress every 10 iterations + :return: (row_indices, coefficients) + """ + if not isinstance(A, np.ndarray): + raise TypeError("argument must be of numpy.ndarray type") + if A.ndim != 2: + raise ValueError("argument must have 2 dimensions") + A = np.asarray(A, dtype=np.float64) + + if _maxvol_func is not None: + return _maxvol_func(A, tol, max_iters, top_k_index, int(verbose)) + + log.warning("compiled pyace.maxvol._maxvol is unavailable (%s); " + "falling back to the slower pure-python maxvol", _import_error) + return py_maxvol(A, tol, max_iters, top_k_index, verbose) diff --git a/src/pyace/maxvol/_maxvol.pyx b/src/pyace/maxvol/_maxvol.pyx new file mode 100644 index 0000000..485c3b2 --- /dev/null +++ b/src/pyace/maxvol/_maxvol.pyx @@ -0,0 +1,127 @@ +#cython: language_level=3 +#cython: embedsignature=True, cdivision=True, boundscheck=False, wraparound=False, initializedcheck=False +""" +float64 1-volume maximisation (maxvol). + +Reduced from maxvolpy 0.3.8 by Alexander Mikhalev, MIT licence -- see +LICENSE.maxvolpy.txt next to this file. Only the float64 ``maxvol`` path is +kept, which is the only one pyace ever calls; the float32/complex variants and +the whole rectangular ``rect_maxvol`` family are gone. + +Two deliberate differences from the original: + +* the pivot-interchange loops use ``range`` rather than ``prange``. Building + those swaps in parallel is a data race -- each swap reads entries earlier + swaps wrote -- and it was only ever safe because nothing passed -fopenmp. +* the scratch buffers are released in a ``finally``, so an error raised by + dgetrf no longer leaks them. +""" +__all__ = ['c_maxvol'] + +import numpy as np +cimport numpy as cnp + +from libc.stdlib cimport malloc, free +from scipy.linalg.cython_blas cimport dtrsm, dcopy, dger +from scipy.linalg.cython_lapack cimport dgetrf + +cnp.import_array() + +cdef extern from "math.h" nogil: + double fabs(double) + + +def c_maxvol(A, tol=1.05, max_iters=100, top_k_index=-1, int verbose=0): + """ + Cython implementation of float64 1-volume maximisation. + + Returns (row_indices, coefficients) exactly as maxvolpy.maxvol.c_maxvol did. + """ + cdef int N, r + cdef cnp.ndarray lu, coef, basis + if not isinstance(A, np.ndarray): + raise TypeError("argument must be of numpy.ndarray type") + if A.ndim != 2: + raise ValueError("argument must have 2 dimensions") + if A.dtype != np.dtype(np.float64): + raise TypeError("argument must be of float64 type, got {}".format(A.dtype)) + N, r = A.shape + if N <= r: + return np.arange(N, dtype=np.int32), np.eye(N, dtype=A.dtype) + if tol < 1: + tol = 1.0 + lu = np.copy(A, order='F') + coef = np.copy(lu, order='F') + basis = np.ndarray(r, dtype=np.int32) + dmaxvol(N, r, lu.data, coef.data, + basis.data, tol, max_iters, top_k_index, verbose) + return basis, coef + + +cdef object dmaxvol(int N, int R, double *lu, double *coef, int *basis, + double tol, int max_iters, int top_k_index, int verbose=0): + cdef int *ipiv = malloc(R * sizeof(int)) + cdef int *interchange = malloc(N * sizeof(int)) + cdef double *tmp_row = malloc(R * sizeof(double)) + cdef double *tmp_column = malloc(N * sizeof(double)) + cdef int info = 0, i, j, tmp_int, i_one = 1, iters = 0 + cdef int k_row, k_col + cdef char cR = b'R', cN = b'N', cU = b'U', cL = b'L' + cdef double d_one = 1, alpha, max_value + cdef double abs_max, tmp + try: + if (ipiv == NULL or interchange == NULL or tmp_row == NULL or + tmp_column == NULL): + raise MemoryError("malloc failed to allocate temporary buffers") + if top_k_index == -1 or top_k_index > N: + top_k_index = N + if top_k_index < R: + top_k_index = R + dgetrf(&top_k_index, &R, lu, &N, ipiv, &info) + if info < 0: + raise ValueError("Internal maxvol_fullrank error, {} argument of" + " dgetrf_ had illegal value".format(info)) + if info > 0: + raise ValueError("Input matrix must not be singular") + for i in range(N): + interchange[i] = i + for i in range(R): + j = ipiv[i] - 1 + if j != i: + tmp_int = interchange[i] + interchange[i] = interchange[j] + interchange[j] = tmp_int + for i in range(R): + basis[i] = interchange[i] + dtrsm(&cR, &cU, &cN, &cN, &N, &R, &d_one, lu, &N, coef, &N) + dtrsm(&cR, &cL, &cN, &cU, &N, &R, &d_one, lu, &N, coef, &N) + while iters < max_iters: + abs_max = -1 + for k_row in range(top_k_index): + for k_col in range(R): + tmp = fabs(coef[k_row + k_col * N]) + if tmp > abs_max: + abs_max = tmp + j = k_row + i = k_col + max_value = coef[j + i * N] + if verbose and iters % 10 == 0: + print('Iter {}/{}: abs_max = {:.3f} (tol = {:.3f})'.format( + iters, max_iters, abs_max, tol)) + if abs_max > tol: + dcopy(&R, coef + j, &N, tmp_row, &i_one) + tmp_row[i] -= d_one + dcopy(&N, coef + i * N, &i_one, tmp_column, &i_one) + basis[i] = j + alpha = (-d_one) / max_value + dger(&N, &R, &alpha, tmp_column, &i_one, tmp_row, &i_one, + coef, &N) + iters += i_one + else: + break + finally: + free(ipiv) + free(interchange) + free(tmp_row) + free(tmp_column) + return From d996100862c90b667706e2934ad67913e117afca Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Fri, 4 Sep 2026 12:18:28 +0200 Subject: [PATCH 11/30] build: drop vendored maxvolpy, require python>=3.9, clamp jobs explicitly lib/maxvolpy is dead now that the float64 kernel is vendored in src/pyace/maxvol; nothing outside the directory referenced it. 7.1 MB gone. python_requires 3.8 -> 3.9, matching grace-tensorpotential's requires-python. It was already wrong in practice: numpy>=2.0, which the maxvol extension compiles against, needs 3.9. default_parallel_jobs() already returned >= 1 on every path, but only implicitly, spread across three branches. Clamp once at the return instead. --- lib/maxvolpy/.gitignore | 4 - lib/maxvolpy/LICENSE.txt | 20 - lib/maxvolpy/MANIFEST.in | 2 - lib/maxvolpy/README.rst | 55 -- lib/maxvolpy/maxvolpy/__init__.py | 16 - lib/maxvolpy/maxvolpy/__version__.py | 1 - lib/maxvolpy/maxvolpy/_maxvol.pyx | 716 -------------------------- lib/maxvolpy/maxvolpy/_maxvol.pyx.src | 294 ----------- lib/maxvolpy/maxvolpy/cross.py | 125 ----- lib/maxvolpy/maxvolpy/maxvol.py | 678 ------------------------ lib/maxvolpy/maxvolpy/misc.py | 84 --- lib/maxvolpy/maxvolpy/setup.py | 26 - lib/maxvolpy/requirements.txt | 4 - lib/maxvolpy/setup.cfg | 2 - lib/maxvolpy/setup.py | 40 -- 15 files changed, 2067 deletions(-) delete mode 100644 lib/maxvolpy/.gitignore delete mode 100644 lib/maxvolpy/LICENSE.txt delete mode 100644 lib/maxvolpy/MANIFEST.in delete mode 100644 lib/maxvolpy/README.rst delete mode 100644 lib/maxvolpy/maxvolpy/__init__.py delete mode 100644 lib/maxvolpy/maxvolpy/__version__.py delete mode 100644 lib/maxvolpy/maxvolpy/_maxvol.pyx delete mode 100644 lib/maxvolpy/maxvolpy/_maxvol.pyx.src delete mode 100644 lib/maxvolpy/maxvolpy/cross.py delete mode 100644 lib/maxvolpy/maxvolpy/maxvol.py delete mode 100644 lib/maxvolpy/maxvolpy/misc.py delete mode 100644 lib/maxvolpy/maxvolpy/setup.py delete mode 100644 lib/maxvolpy/requirements.txt delete mode 100644 lib/maxvolpy/setup.cfg delete mode 100644 lib/maxvolpy/setup.py diff --git a/lib/maxvolpy/.gitignore b/lib/maxvolpy/.gitignore deleted file mode 100644 index 1b79ab0..0000000 --- a/lib/maxvolpy/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -*.pyc -maxvolpy/_maxvol.c -build/* -docs/build/* diff --git a/lib/maxvolpy/LICENSE.txt b/lib/maxvolpy/LICENSE.txt deleted file mode 100644 index e03217d..0000000 --- a/lib/maxvolpy/LICENSE.txt +++ /dev/null @@ -1,20 +0,0 @@ -Copyright (c) 2015 -Alexander Mikhalev - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/lib/maxvolpy/MANIFEST.in b/lib/maxvolpy/MANIFEST.in deleted file mode 100644 index 9211161..0000000 --- a/lib/maxvolpy/MANIFEST.in +++ /dev/null @@ -1,2 +0,0 @@ -include maxvolpy/_maxvol.pyx.src -include LICENSE.txt diff --git a/lib/maxvolpy/README.rst b/lib/maxvolpy/README.rst deleted file mode 100644 index db6d28d..0000000 --- a/lib/maxvolpy/README.rst +++ /dev/null @@ -1,55 +0,0 @@ -About `maxvolpy` -================ - -Module `maxvolpy` is designed for constructing different low-rank -skeleton and cross approximations. There are different strategies of -doing this, but this module is focused on approximations, based on -submatrices of good volume. - -Installation -============ - -Module `maxvolpy` can be installed in 2 ways: - - Pip-installer - - Installation from git repository - -Requirements ------------- - -Package extensively uses `numpy` python package and `blas`/`lapack` -libraries. -Package `numpy` must be of version at least 1.10 - -Pip-installer -------------- - -Simply run `pip` with necessary arguments: - -.. code:: shell - - pip install maxvolpy - -Installation from git repository --------------------------------- - -To compile and install package from repository, you should have -`cython` and `numpy` python packages installed. - -.. code:: shell - - git clone https://bitbucket.org/muxas/maxvolpy - cd maxvolpy - python setup.py install - -Additional Information ----------------------- - -If module was installed from git repository, but was not properly -compiled during installation process due to any errors, it will still -work if module `scipy` is presented in system. Module has backup -functions, written in Python. - -Documentation -============= - -Is available at http://pythonhosted.org/maxvolpy/ \ No newline at end of file diff --git a/lib/maxvolpy/maxvolpy/__init__.py b/lib/maxvolpy/maxvolpy/__init__.py deleted file mode 100644 index cac8c89..0000000 --- a/lib/maxvolpy/maxvolpy/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -Module `maxvolpy` is designed for constructing different low-rank -skeleton and cross approximations. - -Right now, cross approximation techniques are not implemented yet, but -all kinds of algorithms of finding good submatrices to build skeleton -approximations are presented in `maxvol` submodule. What does good -submatrix mean is noted in documentation for `maxvol` submodule. -""" - -from __future__ import absolute_import - -__all__ = ['maxvol'] - -from . import maxvol -from .__version__ import __version__ diff --git a/lib/maxvolpy/maxvolpy/__version__.py b/lib/maxvolpy/maxvolpy/__version__.py deleted file mode 100644 index 17a3314..0000000 --- a/lib/maxvolpy/maxvolpy/__version__.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = '0.3.8' diff --git a/lib/maxvolpy/maxvolpy/_maxvol.pyx b/lib/maxvolpy/maxvolpy/_maxvol.pyx deleted file mode 100644 index f175791..0000000 --- a/lib/maxvolpy/maxvolpy/_maxvol.pyx +++ /dev/null @@ -1,716 +0,0 @@ -#cython: language_level=3 -#cython: embedsignature=True, cdivision=True, boundscheck=False, wraparound=False, initializedcheck=False -__all__ = ['c_maxvol', 'c_rect_maxvol'] -import numpy as np -cimport numpy as cnp -from libc.stdlib cimport malloc, realloc, free -from cython.parallel import prange - -from scipy.linalg.cython_blas cimport ( - strsm, dtrsm, ctrsm, ztrsm, - scopy, ccopy, dcopy, zcopy, - sgemv, dgemv, cgemv, zgemv, - sger, dger, cgerc, cgeru, zgerc, zgeru, - isamax, idamax -) - -from scipy.linalg.cython_lapack cimport ( - sgetrf, dgetrf, cgetrf, zgetrf -) - -cdef extern from "complex.h" nogil: - double cabs(double complex) - float cabsf(float complex) - -cdef extern from "math.h" nogil: - double fabs(double) - float fabsf(float) - -def c_rect_maxvol(A, tol=1., maxK=None, min_add_K=None, minK=None, - start_maxvol_iters=10, identity_submatrix=True, top_k_index=-1): - """ - Cython implementation of rectangular 2-volume maximization. - - For information see `rect_maxvol` function. - """ - cdef int N, r, id_sub - cdef cnp.ndarray lu, coef, basis - if type(A) != np.ndarray: - raise TypeError, "argument must be of numpy.ndarray type" - if len(A.shape) != 2: - raise ValueError, "argument must have 2 dimensions" - N, r = A.shape - if N <= r: - return np.arange(N, dtype=np.int32), np.eye(N, dtype=A.dtype) - lu = np.copy(A, order='F') - if maxK is None or maxK > N: - maxK = N - if maxK < r: - maxK = r - if minK is None or minK < r: - minK = r - if minK > N: - minK = N - if min_add_K is not None: - minK = max(minK, r+min_add_K) - if minK > maxK: - minK = maxK - if identity_submatrix: - id_sub = 1 - else: - id_sub = 0 - try: - if A.dtype is np.dtype(np.float32): - return srect_maxvol(N, r, lu.data, tol, minK, maxK, - start_maxvol_iters, id_sub, top_k_index) - elif A.dtype is np.dtype(np.float64): - return drect_maxvol(N, r, lu.data, tol, minK, maxK, - start_maxvol_iters, id_sub, top_k_index) - elif A.dtype is np.dtype(np.complex64): - return crect_maxvol(N, r, lu.data, tol, minK, - maxK, start_maxvol_iters, id_sub, top_k_index) - elif A.dtype is np.dtype(np.complex128): - return zrect_maxvol(N, r, lu.data, tol, minK, - maxK, start_maxvol_iters, id_sub, top_k_index) - except Exception: - raise - -def c_maxvol(A, tol=1.05, max_iters=100, top_k_index=-1, int verbose=0): - """ - Cython implementation of 1-volume maximization. - - For information see `maxvol` function. - """ - cdef int N, r - cdef cnp.ndarray lu, coef, basis - if type(A) != np.ndarray: - raise TypeError, "argument must be of numpy.ndarray type" - if len(A.shape) != 2: - raise ValueError, "argument must have 2 dimensions" - N, r = A.shape - if N <= r: - return np.arange(N, dtype=np.int32), np.eye(N, dtype=A.dtype) - if tol < 1: - tol = 1.0 - lu = np.copy(A, order='F') - coef = np.copy(lu, order='F') - basis = np.ndarray(r, dtype=np.int32) - try: - if A.dtype is np.dtype(np.float32): - smaxvol(N, r, lu.data, coef.data, - basis.data, tol, max_iters, top_k_index,verbose) - elif A.dtype == np.dtype(np.float64): - dmaxvol(N, r, lu.data, coef.data, - basis.data, tol, max_iters, top_k_index,verbose) - elif A.dtype is np.dtype(np.complex64): - cmaxvol(N, r, lu.data, - coef.data, basis.data, tol, max_iters, top_k_index,verbose) - elif A.dtype is np.dtype(np.complex128): - zmaxvol(N, r, lu.data, - coef.data, basis.data, tol, max_iters, top_k_index,verbose) - else: - raise TypeError("must be of float or complex type") - except Exception: - raise - return basis, coef - - -cdef object srect_maxvol(int N, int R, float *lu, float tol, int minK, - int maxK, int start_maxvol_iters, int identity_submatrix, - int top_k_index,int verbose=0): - cdef char cN = b'N' - cdef int i, j, i_one = 1, K, size = N*R - cdef float d_one = 1.0, d_zero = 0.0, l - cdef float tol2 = tol*tol, tmp, tmp2 - cdef int *basis = malloc(N * sizeof(int)) - cdef float *chosen = malloc(N * sizeof(float)) - cdef int [:]basis_buf - cdef int coef_realloc_step = R, coef_columns = R+coef_realloc_step - cdef float *coef = malloc(N * coef_columns * sizeof(float)) - cdef float *tmp_pointer - cdef float *L = malloc(N * sizeof(float)) - cdef float *V = malloc(N * sizeof(float)) - cdef float *tmp_row = malloc(N * sizeof(float)) - cdef float [:,:] coef_buf - if top_k_index == -1 or top_k_index > N: - top_k_index = N - if top_k_index < R: - top_k_index = R - scopy(&size, lu, &i_one, coef, &i_one) - tmp = 1.05 # tolerance for square maxvol - smaxvol(N, R, lu, coef, basis, tmp, start_maxvol_iters, top_k_index,verbose) - # compute square length for each vector - for j in prange(top_k_index, schedule="static", nogil=True): - L[j] = 0.0 - V[j] = 0.0 - chosen[j] = 1.0 - for i in range(R): - tmp_pointer = coef+i*N - for j in prange(top_k_index, schedule="static", nogil=True): - tmp = fabsf(tmp_pointer[j]) - L[j] += tmp*tmp - for i in prange(R, schedule="static", nogil=True): - L[basis[i]] = 0.0 - chosen[basis[i]] = 0.0 - i = isamax(&top_k_index, L, &i_one)-1 - K = R - while K < minK or (L[i] > tol2 and K < maxK): - basis[K] = i - chosen[i] = 0.0 - #scopy(&K, coef+i, &N, tmp_row, &i_one) - tmp_pointer = coef+i - for j in prange(K, schedule="static", nogil=True): - tmp_row[j] = tmp_pointer[j*N] - sgemv(&cN, &N, &K, &d_one, coef, &N, tmp_row, &i_one, &d_zero, V, - &i_one) - l = (-d_one)/(1+V[i]) - sger(&N, &K, &l, V, &i_one, tmp_row, &i_one, coef, &N) - tmp = -l - if coef_columns <= K: - coef_columns += coef_realloc_step - coef = realloc(coef, N * coef_columns * sizeof(float)) - tmp_pointer = coef+K*N - for j in prange(N, schedule="static", nogil=True): - tmp_pointer[j] = tmp*V[j] - for j in prange(top_k_index, schedule="static", nogil=True): - tmp2 = fabsf(V[j]) - L[j] -= tmp2*tmp2*tmp - L[j] *= chosen[j] - i = isamax(&top_k_index, L, &i_one)-1 - K += 1 - free(L) - free(V) - free(tmp_row) - C = np.ndarray((N, K), order='F', dtype=np.float32) - coef_buf = C - for i in prange(K, schedule="static", nogil=True): - for j in range(N): - coef_buf[j, i] = coef[i*N+j] - free(coef) - if identity_submatrix == 1: - for i in prange(K, schedule="static", nogil=True): - tmp_pointer = &coef_buf[0, 0]+basis[i] - for j in range(K): - tmp_pointer[j*N] = 0.0 - tmp_pointer[i*N] = 1.0 - I = np.ndarray(K, dtype=np.int32) - basis_buf = I - for i in prange(K, schedule="static", nogil=True): - basis_buf[i] = basis[i] - free(basis) - return I, C - -cdef object smaxvol(int N, int R, float *lu, float *coef, int *basis, - float tol, int max_iters, int top_k_index, int verbose=0): - cdef int *ipiv = malloc(R * sizeof(int)) - cdef int *interchange = malloc(N * sizeof(int)) - cdef float *tmp_row = malloc(R*sizeof(float)) - cdef float *tmp_column = malloc(N*sizeof(float)) - cdef int info = 0, size = N * R, i, j, tmp_int, i_one = 1, iters = 0 - cdef int k_row, k_col - cdef char cR = b'R', cN = b'N', cU = b'U', cL = b'L' - cdef float d_one = 1, alpha, max_value - cdef float abs_max, tmp - if (ipiv == NULL or interchange == NULL or tmp_row == NULL or - tmp_column == NULL): - raise MemoryError("malloc failed to allocate temporary buffers") - if top_k_index == -1 or top_k_index > N: - top_k_index = N - if top_k_index < R: - top_k_index = R - sgetrf(&top_k_index, &R, lu, &N, ipiv, &info) - if info < 0: - raise ValueError("Internal maxvol_fullrank error, {} argument of" - " sgetrf_ had illegal value".format(info)) - if info > 0: - raise ValueError("Input matrix must not be singular") - for i in prange(N, schedule="static", nogil=True): - interchange[i] = i - for i in prange(R, schedule="static", nogil=True): - j = ipiv[i]-1 - if j != i: - tmp_int = interchange[i] - interchange[i] = interchange[j] - interchange[j] = tmp_int - free(ipiv) - for i in prange(R, schedule="static", nogil=True): - basis[i] = interchange[i] - free(interchange) - strsm(&cR, &cU, &cN, &cN, &N, &R, &d_one, lu, &N, coef, &N) - strsm(&cR, &cL, &cN, &cU, &N, &R, &d_one, lu, &N, coef, &N) - while iters < max_iters: - abs_max = -1 - for k_row in range(top_k_index): - for k_col in range(R): - tmp = fabsf(coef[k_row+k_col*N]) - if tmp > abs_max: - abs_max = tmp - j = k_row - i = k_col - max_value = coef[j+i*N] - if verbose and iters % 10 == 0: - print('Iter {}/{}: abs_max = {:.3f} (tol = {:.3f})'.format(iters, max_iters, abs_max, tol)) - if abs_max > tol: - scopy(&R, coef+j, &N, tmp_row, &i_one) - tmp_row[i] -= d_one - scopy(&N, coef+i*N, &i_one, tmp_column, &i_one) - basis[i] = j - alpha = (-d_one)/max_value - sger(&N, &R, &alpha, tmp_column, &i_one, tmp_row, &i_one, - coef, &N) - iters += i_one - else: - break - free(tmp_row) - free(tmp_column) - return - -cdef object drect_maxvol(int N, int R, double *lu, double tol, int minK, - int maxK, int start_maxvol_iters, int identity_submatrix, - int top_k_index, int verbose=0): - cdef char cN = b'N' - cdef int i, j, i_one = 1, K, size = N*R - cdef double d_one = 1.0, d_zero = 0.0, l - cdef double tol2 = tol*tol, tmp, tmp2 - cdef int *basis = malloc(N * sizeof(int)) - cdef double *chosen = malloc(N * sizeof(double)) - cdef int [:]basis_buf - cdef int coef_realloc_step = R, coef_columns = R+coef_realloc_step - cdef double *coef = malloc(N * coef_columns * sizeof(double)) - cdef double *tmp_pointer - cdef double *L = malloc(N * sizeof(double)) - cdef double *V = malloc(N * sizeof(double)) - cdef double *tmp_row = malloc(N * sizeof(double)) - cdef double [:,:] coef_buf - if top_k_index == -1 or top_k_index > N: - top_k_index = N - if top_k_index < R: - top_k_index = R - dcopy(&size, lu, &i_one, coef, &i_one) - tmp = 1.05 # tolerance for square maxvol - dmaxvol(N, R, lu, coef, basis, tmp, start_maxvol_iters, top_k_index, verbose) - # compute square length for each vector - for j in prange(top_k_index, schedule="static", nogil=True): - L[j] = 0.0 - V[j] = 0.0 - chosen[j] = 1.0 - for i in range(R): - tmp_pointer = coef+i*N - for j in prange(top_k_index, schedule="static", nogil=True): - tmp = fabs(tmp_pointer[j]) - L[j] += tmp*tmp - for i in prange(R, schedule="static", nogil=True): - L[basis[i]] = 0.0 - chosen[basis[i]] = 0.0 - i = idamax(&top_k_index, L, &i_one)-1 - K = R - while K < minK or (L[i] > tol2 and K < maxK): - basis[K] = i - chosen[i] = 0.0 - #dcopy(&K, coef+i, &N, tmp_row, &i_one) - tmp_pointer = coef+i - for j in prange(K, schedule="static", nogil=True): - tmp_row[j] = tmp_pointer[j*N] - dgemv(&cN, &N, &K, &d_one, coef, &N, tmp_row, &i_one, &d_zero, V, - &i_one) - l = (-d_one)/(1+V[i]) - dger(&N, &K, &l, V, &i_one, tmp_row, &i_one, coef, &N) - tmp = -l - if coef_columns <= K: - coef_columns += coef_realloc_step - coef = realloc(coef, N * coef_columns * sizeof(double)) - tmp_pointer = coef+K*N - for j in prange(N, schedule="static", nogil=True): - tmp_pointer[j] = tmp*V[j] - for j in prange(top_k_index, schedule="static", nogil=True): - tmp2 = fabs(V[j]) - L[j] -= tmp2*tmp2*tmp - L[j] *= chosen[j] - i = idamax(&top_k_index, L, &i_one)-1 - K += 1 - free(L) - free(V) - free(tmp_row) - C = np.ndarray((N, K), order='F', dtype=np.float64) - coef_buf = C - for i in prange(K, schedule="static", nogil=True): - for j in range(N): - coef_buf[j, i] = coef[i*N+j] - free(coef) - if identity_submatrix == 1: - for i in prange(K, schedule="static", nogil=True): - tmp_pointer = &coef_buf[0, 0]+basis[i] - for j in range(K): - tmp_pointer[j*N] = 0.0 - tmp_pointer[i*N] = 1.0 - I = np.ndarray(K, dtype=np.int32) - basis_buf = I - for i in prange(K, schedule="static", nogil=True): - basis_buf[i] = basis[i] - free(basis) - return I, C - -cdef object dmaxvol(int N, int R, double *lu, double *coef, int *basis, - double tol, int max_iters, int top_k_index, int verbose=0): - cdef int *ipiv = malloc(R * sizeof(int)) - cdef int *interchange = malloc(N * sizeof(int)) - cdef double *tmp_row = malloc(R*sizeof(double)) - cdef double *tmp_column = malloc(N*sizeof(double)) - cdef int info = 0, size = N * R, i, j, tmp_int, i_one = 1, iters = 0 - cdef int k_row, k_col - cdef char cR = b'R', cN = b'N', cU = b'U', cL = b'L' - cdef double d_one = 1, alpha, max_value - cdef double abs_max, tmp - if (ipiv == NULL or interchange == NULL or tmp_row == NULL or - tmp_column == NULL): - raise MemoryError("malloc failed to allocate temporary buffers") - if top_k_index == -1 or top_k_index > N: - top_k_index = N - if top_k_index < R: - top_k_index = R - dgetrf(&top_k_index, &R, lu, &N, ipiv, &info) - if info < 0: - raise ValueError("Internal maxvol_fullrank error, {} argument of" - " dgetrf_ had illegal value".format(info)) - if info > 0: - raise ValueError("Input matrix must not be singular") - for i in prange(N, schedule="static", nogil=True): - interchange[i] = i - for i in prange(R, schedule="static", nogil=True): - j = ipiv[i]-1 - if j != i: - tmp_int = interchange[i] - interchange[i] = interchange[j] - interchange[j] = tmp_int - free(ipiv) - for i in prange(R, schedule="static", nogil=True): - basis[i] = interchange[i] - free(interchange) - dtrsm(&cR, &cU, &cN, &cN, &N, &R, &d_one, lu, &N, coef, &N) - dtrsm(&cR, &cL, &cN, &cU, &N, &R, &d_one, lu, &N, coef, &N) - while iters < max_iters: - abs_max = -1 - for k_row in range(top_k_index): - for k_col in range(R): - tmp = fabs(coef[k_row+k_col*N]) - if tmp > abs_max: - abs_max = tmp - j = k_row - i = k_col - max_value = coef[j+i*N] - if verbose and iters % 10 == 0: - print('Iter {}/{}: abs_max = {:.3f} (tol = {:.3f})'.format(iters, max_iters, abs_max, tol)) - if abs_max > tol: - dcopy(&R, coef+j, &N, tmp_row, &i_one) - tmp_row[i] -= d_one - dcopy(&N, coef+i*N, &i_one, tmp_column, &i_one) - basis[i] = j - alpha = (-d_one)/max_value - dger(&N, &R, &alpha, tmp_column, &i_one, tmp_row, &i_one, - coef, &N) - iters += i_one - else: - break - free(tmp_row) - free(tmp_column) - return - -cdef object crect_maxvol(int N, int R, float complex *lu, float tol, int minK, - int maxK, int start_maxvol_iters, int identity_submatrix, - int top_k_index, int verbose=0): - cdef char cN = b'N' - cdef int i, j, i_one = 1, K, size = N*R - cdef float complex d_one = 1.0, d_zero = 0.0, l - cdef float tol2 = tol*tol, tmp, tmp2 - cdef int *basis = malloc(N * sizeof(int)) - cdef float *chosen = malloc(N * sizeof(float)) - cdef int [:]basis_buf - cdef int coef_realloc_step = R, coef_columns = R+coef_realloc_step - cdef float complex *coef = malloc(N * coef_columns * sizeof(float complex)) - cdef float complex *tmp_pointer - cdef float *L = malloc(N * sizeof(float)) - cdef float complex *V = malloc(N * sizeof(float complex)) - cdef float complex *tmp_row = malloc(N * sizeof(float complex)) - cdef float complex [:,:] coef_buf - if top_k_index == -1 or top_k_index > N: - top_k_index = N - if top_k_index < R: - top_k_index = R - ccopy(&size, lu, &i_one, coef, &i_one) - tmp = 1.05 # tolerance for square maxvol - cmaxvol(N, R, lu, coef, basis, tmp, start_maxvol_iters, top_k_index, verbose) - # compute square length for each vector - for j in prange(top_k_index, schedule="static", nogil=True): - L[j] = 0.0 - V[j] = 0.0 - chosen[j] = 1.0 - for i in range(R): - tmp_pointer = coef+i*N - for j in prange(top_k_index, schedule="static", nogil=True): - tmp = cabsf(tmp_pointer[j]) - L[j] += tmp*tmp - for i in prange(R, schedule="static", nogil=True): - L[basis[i]] = 0.0 - chosen[basis[i]] = 0.0 - i = isamax(&top_k_index, L, &i_one)-1 - K = R - while K < minK or (L[i] > tol2 and K < maxK): - basis[K] = i - chosen[i] = 0.0 - #ccopy(&K, coef+i, &N, tmp_row, &i_one) - tmp_pointer = coef+i - for j in prange(K, schedule="static", nogil=True): - tmp_row[j] = tmp_pointer[j*N].conjugate() - cgemv(&cN, &N, &K, &d_one, coef, &N, tmp_row, &i_one, &d_zero, V, - &i_one) - l = (-d_one)/(1+V[i]) - cgerc(&N, &K, &l, V, &i_one, tmp_row, &i_one, coef, &N) - tmp = -l.real - if coef_columns <= K: - coef_columns += coef_realloc_step - coef = realloc(coef, N * coef_columns * sizeof(float complex)) - tmp_pointer = coef+K*N - for j in prange(N, schedule="static", nogil=True): - tmp_pointer[j] = tmp*V[j] - for j in prange(top_k_index, schedule="static", nogil=True): - tmp2 = cabsf(V[j]) - L[j] -= tmp2*tmp2*tmp - L[j] *= chosen[j] - i = isamax(&top_k_index, L, &i_one)-1 - K += 1 - free(L) - free(V) - free(tmp_row) - C = np.ndarray((N, K), order='F', dtype=np.complex64) - coef_buf = C - for i in prange(K, schedule="static", nogil=True): - for j in range(N): - coef_buf[j, i] = coef[i*N+j] - free(coef) - if identity_submatrix == 1: - for i in prange(K, schedule="static", nogil=True): - tmp_pointer = &coef_buf[0, 0]+basis[i] - for j in range(K): - tmp_pointer[j*N] = 0.0 - tmp_pointer[i*N] = 1.0 - I = np.ndarray(K, dtype=np.int32) - basis_buf = I - for i in prange(K, schedule="static", nogil=True): - basis_buf[i] = basis[i] - free(basis) - return I, C - -cdef object cmaxvol(int N, int R, float complex *lu, float complex *coef, int *basis, - float tol, int max_iters, int top_k_index, int verbose=0): - cdef int *ipiv = malloc(R * sizeof(int)) - cdef int *interchange = malloc(N * sizeof(int)) - cdef float complex *tmp_row = malloc(R*sizeof(float complex)) - cdef float complex *tmp_column = malloc(N*sizeof(float complex)) - cdef int info = 0, size = N * R, i, j, tmp_int, i_one = 1, iters = 0 - cdef int k_row, k_col - cdef char cR = b'R', cN = b'N', cU = b'U', cL = b'L' - cdef float complex d_one = 1, alpha, max_value - cdef float abs_max, tmp - if (ipiv == NULL or interchange == NULL or tmp_row == NULL or - tmp_column == NULL): - raise MemoryError("malloc failed to allocate temporary buffers") - if top_k_index == -1 or top_k_index > N: - top_k_index = N - if top_k_index < R: - top_k_index = R - cgetrf(&top_k_index, &R, lu, &N, ipiv, &info) - if info < 0: - raise ValueError("Internal maxvol_fullrank error, {} argument of" - " cgetrf_ had illegal value".format(info)) - if info > 0: - raise ValueError("Input matrix must not be singular") - for i in prange(N, schedule="static", nogil=True): - interchange[i] = i - for i in prange(R, schedule="static", nogil=True): - j = ipiv[i]-1 - if j != i: - tmp_int = interchange[i] - interchange[i] = interchange[j] - interchange[j] = tmp_int - free(ipiv) - for i in prange(R, schedule="static", nogil=True): - basis[i] = interchange[i] - free(interchange) - ctrsm(&cR, &cU, &cN, &cN, &N, &R, &d_one, lu, &N, coef, &N) - ctrsm(&cR, &cL, &cN, &cU, &N, &R, &d_one, lu, &N, coef, &N) - while iters < max_iters: - abs_max = -1 - for k_row in range(top_k_index): - for k_col in range(R): - tmp = cabsf(coef[k_row+k_col*N]) - if tmp > abs_max: - abs_max = tmp - j = k_row - i = k_col - max_value = coef[j+i*N] - if verbose and iters % 10 == 0: - print('Iter {}/{}: abs_max = {:.3f} (tol = {:.3f})'.format(iters, max_iters, abs_max, tol)) - if abs_max > tol: - ccopy(&R, coef+j, &N, tmp_row, &i_one) - tmp_row[i] -= d_one - ccopy(&N, coef+i*N, &i_one, tmp_column, &i_one) - basis[i] = j - alpha = (-d_one)/max_value - cgeru(&N, &R, &alpha, tmp_column, &i_one, tmp_row, &i_one, - coef, &N) - iters += i_one - else: - break - free(tmp_row) - free(tmp_column) - return - -cdef object zrect_maxvol(int N, int R, double complex *lu, double tol, int minK, - int maxK, int start_maxvol_iters, int identity_submatrix, - int top_k_index, int verbose=0): - cdef char cN = b'N' - cdef int i, j, i_one = 1, K, size = N*R - cdef double complex d_one = 1.0, d_zero = 0.0, l - cdef double tol2 = tol*tol, tmp, tmp2 - cdef int *basis = malloc(N * sizeof(int)) - cdef double *chosen = malloc(N * sizeof(double)) - cdef int [:]basis_buf - cdef int coef_realloc_step = R, coef_columns = R+coef_realloc_step - cdef double complex *coef = malloc(N * coef_columns * sizeof(double complex)) - cdef double complex *tmp_pointer - cdef double *L = malloc(N * sizeof(double)) - cdef double complex *V = malloc(N * sizeof(double complex)) - cdef double complex *tmp_row = malloc(N * sizeof(double complex)) - cdef double complex [:,:] coef_buf - if top_k_index == -1 or top_k_index > N: - top_k_index = N - if top_k_index < R: - top_k_index = R - zcopy(&size, lu, &i_one, coef, &i_one) - tmp = 1.05 # tolerance for square maxvol - zmaxvol(N, R, lu, coef, basis, tmp, start_maxvol_iters, top_k_index, verbose) - # compute square length for each vector - for j in prange(top_k_index, schedule="static", nogil=True): - L[j] = 0.0 - V[j] = 0.0 - chosen[j] = 1.0 - for i in range(R): - tmp_pointer = coef+i*N - for j in prange(top_k_index, schedule="static", nogil=True): - tmp = cabs(tmp_pointer[j]) - L[j] += tmp*tmp - for i in prange(R, schedule="static", nogil=True): - L[basis[i]] = 0.0 - chosen[basis[i]] = 0.0 - i = idamax(&top_k_index, L, &i_one)-1 - K = R - while K < minK or (L[i] > tol2 and K < maxK): - basis[K] = i - chosen[i] = 0.0 - #zcopy(&K, coef+i, &N, tmp_row, &i_one) - tmp_pointer = coef+i - for j in prange(K, schedule="static", nogil=True): - tmp_row[j] = tmp_pointer[j*N].conjugate() - zgemv(&cN, &N, &K, &d_one, coef, &N, tmp_row, &i_one, &d_zero, V, - &i_one) - l = (-d_one)/(1+V[i]) - zgerc(&N, &K, &l, V, &i_one, tmp_row, &i_one, coef, &N) - tmp = -l.real - if coef_columns <= K: - coef_columns += coef_realloc_step - coef = realloc(coef, N * coef_columns * sizeof(double complex)) - tmp_pointer = coef+K*N - for j in prange(N, schedule="static", nogil=True): - tmp_pointer[j] = tmp*V[j] - for j in prange(top_k_index, schedule="static", nogil=True): - tmp2 = cabs(V[j]) - L[j] -= tmp2*tmp2*tmp - L[j] *= chosen[j] - i = idamax(&top_k_index, L, &i_one)-1 - K += 1 - free(L) - free(V) - free(tmp_row) - C = np.ndarray((N, K), order='F', dtype=np.complex128) - coef_buf = C - for i in prange(K, schedule="static", nogil=True): - for j in range(N): - coef_buf[j, i] = coef[i*N+j] - free(coef) - if identity_submatrix == 1: - for i in prange(K, schedule="static", nogil=True): - tmp_pointer = &coef_buf[0, 0]+basis[i] - for j in range(K): - tmp_pointer[j*N] = 0.0 - tmp_pointer[i*N] = 1.0 - I = np.ndarray(K, dtype=np.int32) - basis_buf = I - for i in prange(K, schedule="static", nogil=True): - basis_buf[i] = basis[i] - free(basis) - return I, C - -cdef object zmaxvol(int N, int R, double complex *lu, double complex *coef, int *basis, - double tol, int max_iters, int top_k_index, int verbose=0): - cdef int *ipiv = malloc(R * sizeof(int)) - cdef int *interchange = malloc(N * sizeof(int)) - cdef double complex *tmp_row = malloc(R*sizeof(double complex)) - cdef double complex *tmp_column = malloc(N*sizeof(double complex)) - cdef int info = 0, size = N * R, i, j, tmp_int, i_one = 1, iters = 0 - cdef int k_row, k_col - cdef char cR = b'R', cN = b'N', cU = b'U', cL = b'L' - cdef double complex d_one = 1, alpha, max_value - cdef double abs_max, tmp - if (ipiv == NULL or interchange == NULL or tmp_row == NULL or - tmp_column == NULL): - raise MemoryError("malloc failed to allocate temporary buffers") - if top_k_index == -1 or top_k_index > N: - top_k_index = N - if top_k_index < R: - top_k_index = R - zgetrf(&top_k_index, &R, lu, &N, ipiv, &info) - if info < 0: - raise ValueError("Internal maxvol_fullrank error, {} argument of" - " zgetrf_ had illegal value".format(info)) - if info > 0: - raise ValueError("Input matrix must not be singular") - for i in prange(N, schedule="static", nogil=True): - interchange[i] = i - for i in prange(R, schedule="static", nogil=True): - j = ipiv[i]-1 - if j != i: - tmp_int = interchange[i] - interchange[i] = interchange[j] - interchange[j] = tmp_int - free(ipiv) - for i in prange(R, schedule="static", nogil=True): - basis[i] = interchange[i] - free(interchange) - ztrsm(&cR, &cU, &cN, &cN, &N, &R, &d_one, lu, &N, coef, &N) - ztrsm(&cR, &cL, &cN, &cU, &N, &R, &d_one, lu, &N, coef, &N) - while iters < max_iters: - abs_max = -1 - for k_row in range(top_k_index): - for k_col in range(R): - tmp = cabs(coef[k_row+k_col*N]) - if tmp > abs_max: - abs_max = tmp - j = k_row - i = k_col - max_value = coef[j+i*N] - if verbose and iters % 10 == 0: - print('Iter {}/{}: abs_max = {:.3f} (tol = {:.3f})'.format(iters, max_iters, abs_max, tol)) - if abs_max > tol: - zcopy(&R, coef+j, &N, tmp_row, &i_one) - tmp_row[i] -= d_one - zcopy(&N, coef+i*N, &i_one, tmp_column, &i_one) - basis[i] = j - alpha = (-d_one)/max_value - zgeru(&N, &R, &alpha, tmp_column, &i_one, tmp_row, &i_one, - coef, &N) - iters += i_one - else: - break - free(tmp_row) - free(tmp_column) - return diff --git a/lib/maxvolpy/maxvolpy/_maxvol.pyx.src b/lib/maxvolpy/maxvolpy/_maxvol.pyx.src deleted file mode 100644 index abf7097..0000000 --- a/lib/maxvolpy/maxvolpy/_maxvol.pyx.src +++ /dev/null @@ -1,294 +0,0 @@ -python_code = """ -#cython: embedsignature=True, cdivision=True, boundscheck=False, \ -wraparound=False, initializedcheck=False -__all__ = ['c_maxvol', 'c_rect_maxvol'] -import numpy as np -cimport numpy as cnp -from libc.stdlib cimport malloc, realloc, free -from cython.parallel import prange - -from scipy.linalg.cython_blas cimport ( - strsm, dtrsm, ctrsm, ztrsm, - scopy, ccopy, dcopy, zcopy, - sgemv, dgemv, cgemv, zgemv, - sger, dger, cgerc, cgeru, zgerc, zgeru, - isamax, idamax -) - -from scipy.linalg.cython_lapack cimport ( - sgetrf, dgetrf, cgetrf, zgetrf -) - -cdef extern from "complex.h" nogil: - double cabs(double complex) - float cabsf(float complex) - -cdef extern from "math.h" nogil: - double fabs(double) - float fabsf(float) - -def c_rect_maxvol(A, tol=1., maxK=None, min_add_K=None, minK=None, - start_maxvol_iters=10, identity_submatrix=True, top_k_index=-1): - \"\"\" - Cython implementation of rectangular 2-volume maximization. - - For information see `rect_maxvol` function. - \"\"\" - cdef int N, r, id_sub - cdef cnp.ndarray lu, coef, basis - if type(A) != np.ndarray: - raise TypeError, "argument must be of numpy.ndarray type" - if len(A.shape) != 2: - raise ValueError, "argument must have 2 dimensions" - N, r = A.shape - if N <= r: - return np.arange(N, dtype=np.int32), np.eye(N, dtype=A.dtype) - lu = np.copy(A, order='F') - if maxK is None or maxK > N: - maxK = N - if maxK < r: - maxK = r - if minK is None or minK < r: - minK = r - if minK > N: - minK = N - if min_add_K is not None: - minK = max(minK, r+min_add_K) - if minK > maxK: - minK = maxK - if identity_submatrix: - id_sub = 1 - else: - id_sub = 0 - try: - if A.dtype is np.dtype(np.float32): - return srect_maxvol(N, r, lu.data, tol, minK, maxK, - start_maxvol_iters, id_sub, top_k_index) - elif A.dtype is np.dtype(np.float64): - return drect_maxvol(N, r, lu.data, tol, minK, maxK, - start_maxvol_iters, id_sub, top_k_index) - elif A.dtype is np.dtype(np.complex64): - return crect_maxvol(N, r, lu.data, tol, minK, - maxK, start_maxvol_iters, id_sub, top_k_index) - elif A.dtype is np.dtype(np.complex128): - return zrect_maxvol(N, r, lu.data, tol, minK, - maxK, start_maxvol_iters, id_sub, top_k_index) - except Exception: - raise - -def c_maxvol(A, tol=1.05, max_iters=100, top_k_index=-1): - \"\"\" - Cython implementation of 1-volume maximization. - - For information see `maxvol` function. - \"\"\" - cdef int N, r - cdef cnp.ndarray lu, coef, basis - if type(A) != np.ndarray: - raise TypeError, "argument must be of numpy.ndarray type" - if len(A.shape) != 2: - raise ValueError, "argument must have 2 dimensions" - N, r = A.shape - if N <= r: - return np.arange(N, dtype=np.int32), np.eye(N, dtype=A.dtype) - if tol < 1: - tol = 1.0 - lu = np.copy(A, order='F') - coef = np.copy(lu, order='F') - basis = np.ndarray(r, dtype=np.int32) - try: - if A.dtype is np.dtype(np.float32): - smaxvol(N, r, lu.data, coef.data, - basis.data, tol, max_iters, top_k_index) - elif A.dtype == np.dtype(np.float64): - dmaxvol(N, r, lu.data, coef.data, - basis.data, tol, max_iters, top_k_index) - elif A.dtype is np.dtype(np.complex64): - cmaxvol(N, r, lu.data, - coef.data, basis.data, tol, max_iters, top_k_index) - elif A.dtype is np.dtype(np.complex128): - zmaxvol(N, r, lu.data, - coef.data, basis.data, tol, max_iters, top_k_index) - else: - raise TypeError("must be of float or complex type") - except Exception: - raise - return basis, coef - -""" - -maxvol_template = """ -cdef object ${D}rect_maxvol(int N, int R, $dtype *lu, $dtype2 tol, int minK, - int maxK, int start_maxvol_iters, int identity_submatrix, - int top_k_index): - cdef char cN = 'N' - cdef int i, j, i_one = 1, K, size = N*R - cdef $dtype d_one = 1.0, d_zero = 0.0, l - cdef $dtype2 tol2 = tol*tol, tmp, tmp2 - cdef int *basis = malloc(N * sizeof(int)) - cdef $dtype2 *chosen = <$dtype2 *> malloc(N * sizeof($dtype2)) - cdef int [:]basis_buf - cdef int coef_realloc_step = R, coef_columns = R+coef_realloc_step - cdef $dtype *coef = <$dtype *> malloc(N * coef_columns * sizeof($dtype)) - cdef $dtype *tmp_pointer - cdef $dtype2 *L = <$dtype2 *> malloc(N * sizeof($dtype2)) - cdef $dtype *V = <$dtype *> malloc(N * sizeof($dtype)) - cdef $dtype *tmp_row = <$dtype *> malloc(N * sizeof($dtype)) - cdef $dtype [:,:] coef_buf - if top_k_index == -1 or top_k_index > N: - top_k_index = N - if top_k_index < R: - top_k_index = R - ${D}copy(&size, lu, &i_one, coef, &i_one) - tmp = 1.05 # tolerance for square maxvol - ${D}maxvol(N, R, lu, coef, basis, tmp, start_maxvol_iters, top_k_index) - # compute square length for each vector - for j in prange(top_k_index, schedule="static", nogil=True): - L[j] = 0.0 - V[j] = 0.0 - chosen[j] = 1.0 - for i in range(R): - tmp_pointer = coef+i*N - for j in prange(top_k_index, schedule="static", nogil=True): - tmp = ${abs}(tmp_pointer[j]) - L[j] += tmp*tmp - for i in prange(R, schedule="static", nogil=True): - L[basis[i]] = 0.0 - chosen[basis[i]] = 0.0 - i = i${D2}amax(&top_k_index, L, &i_one)-1 - K = R - while K < minK or (L[i] > tol2 and K < maxK): - basis[K] = i - chosen[i] = 0.0 - #${D}copy(&K, coef+i, &N, tmp_row, &i_one) - tmp_pointer = coef+i - for j in prange(K, schedule="static", nogil=True): - tmp_row[j] = tmp_pointer[j*N]$conj - ${D}gemv(&cN, &N, &K, &d_one, coef, &N, tmp_row, &i_one, &d_zero, V, - &i_one) - l = (-d_one)/(1+V[i]) - $gerc(&N, &K, &l, V, &i_one, tmp_row, &i_one, coef, &N) - tmp = -l$dotreal - if coef_columns <= K: - coef_columns += coef_realloc_step - coef = <$dtype *> realloc(coef, N * coef_columns * sizeof($dtype)) - tmp_pointer = coef+K*N - for j in prange(N, schedule="static", nogil=True): - tmp_pointer[j] = tmp*V[j] - for j in prange(top_k_index, schedule="static", nogil=True): - tmp2 = ${abs}(V[j]) - L[j] -= tmp2*tmp2*tmp - L[j] *= chosen[j] - i = i${D2}amax(&top_k_index, L, &i_one)-1 - K += 1 - free(L) - free(V) - free(tmp_row) - C = np.ndarray((N, K), order='F', dtype=$npdtype) - coef_buf = C - for i in prange(K, schedule="static", nogil=True): - for j in range(N): - coef_buf[j, i] = coef[i*N+j] - free(coef) - if identity_submatrix == 1: - for i in prange(K, schedule="static", nogil=True): - tmp_pointer = &coef_buf[0, 0]+basis[i] - for j in range(K): - tmp_pointer[j*N] = 0.0 - tmp_pointer[i*N] = 1.0 - I = np.ndarray(K, dtype=np.int32) - basis_buf = I - for i in prange(K, schedule="static", nogil=True): - basis_buf[i] = basis[i] - free(basis) - return I, C - -cdef object ${D}maxvol(int N, int R, $dtype *lu, $dtype *coef, int *basis, - $dtype2 tol, int max_iters, int top_k_index): - cdef int *ipiv = malloc(R * sizeof(int)) - cdef int *interchange = malloc(N * sizeof(int)) - cdef $dtype *tmp_row = <$dtype *> malloc(R*sizeof($dtype)) - cdef $dtype *tmp_column = <$dtype *> malloc(N*sizeof($dtype)) - cdef int info = 0, size = N * R, i, j, tmp_int, i_one = 1, iters = 0 - cdef int k_row, k_col - cdef char cR = 'R', cN = 'N', cU = 'U', cL = 'L' - cdef $dtype d_one = 1, alpha, max_value - cdef $dtype2 abs_max, tmp - if (ipiv == NULL or interchange == NULL or tmp_row == NULL or - tmp_column == NULL): - raise MemoryError("malloc failed to allocate temporary buffers") - if top_k_index == -1 or top_k_index > N: - top_k_index = N - if top_k_index < R: - top_k_index = R - ${D}getrf(&top_k_index, &R, lu, &N, ipiv, &info) - if info < 0: - raise ValueError("Internal maxvol_fullrank error, {} argument of" - " ${D}getrf_ had illegal value".format(info)) - if info > 0: - raise ValueError("Input matrix must not be singular") - for i in prange(N, schedule="static", nogil=True): - interchange[i] = i - for i in prange(R, schedule="static", nogil=True): - j = ipiv[i]-1 - if j != i: - tmp_int = interchange[i] - interchange[i] = interchange[j] - interchange[j] = tmp_int - free(ipiv) - for i in prange(R, schedule="static", nogil=True): - basis[i] = interchange[i] - free(interchange) - ${D}trsm(&cR, &cU, &cN, &cN, &N, &R, &d_one, lu, &N, coef, &N) - ${D}trsm(&cR, &cL, &cN, &cU, &N, &R, &d_one, lu, &N, coef, &N) - while iters < max_iters: - abs_max = -1 - for k_row in range(top_k_index): - for k_col in range(R): - tmp = ${abs}(coef[k_row+k_col*N]) - if tmp > abs_max: - abs_max = tmp - j = k_row - i = k_col - max_value = coef[j+i*N] - if abs_max > tol: - ${D}copy(&R, coef+j, &N, tmp_row, &i_one) - tmp_row[i] -= d_one - ${D}copy(&N, coef+i*N, &i_one, tmp_column, &i_one) - basis[i] = j - alpha = (-d_one)/max_value - $geru(&N, &R, &alpha, tmp_column, &i_one, tmp_row, &i_one, - coef, &N) - iters += i_one - else: - break - free(tmp_row) - free(tmp_column) - return -""" - - -from string import Template - -code_maxvol = Template(maxvol_template) - -code_float = code_maxvol.substitute(D='s', D2='s', gerc='sger', - geru='sger', dtype='float', dtype2='float', npdtype='np.float32', - dotreal='', conj='', abs='fabsf') -code_double = code_maxvol.substitute(D='d', D2='d', gerc='dger', - geru='dger', dtype='double', dtype2='double', npdtype='np.float64', - dotreal='', conj='', abs='fabs') -code_floatcomplex = code_maxvol.substitute(D='c', D2='s', gerc='cgerc', - geru='cgeru', dtype='float complex', dtype2='float', - npdtype='np.complex64', dotreal='.real', conj='.conjugate()', abs='cabsf') -code_doublecomplex = code_maxvol.substitute(D='z', D2='d', gerc='zgerc', - geru='zgeru', dtype='double complex', dtype2='double', - npdtype='np.complex128', dotreal='.real', conj='.conjugate()', abs='cabs') - -f = open('_maxvol.pyx', 'w') -f.write(python_code) -f.write(code_float) -f.write(code_double) -f.write(code_floatcomplex) -f.write(code_doublecomplex) -f.close() diff --git a/lib/maxvolpy/maxvolpy/cross.py b/lib/maxvolpy/maxvolpy/cross.py deleted file mode 100644 index 9eeff74..0000000 --- a/lib/maxvolpy/maxvolpy/cross.py +++ /dev/null @@ -1,125 +0,0 @@ -from __future__ import absolute_import, division, print_function -import numpy as np - -def py_cross_full(A, rank): - """returns cross approximation of matrix A, computed with full pivoting.""" - rows = np.ndarray(A.shape[0], dtype = np.int32) - cols = np.ndarray(A.shape[1], dtype = np.int32) - -def py_lu_full_fixed(A, rank): - N, M = A.shape - U = np.ndarray((N, rank), dtype = A.dtype) - V = np.ndarray((rank, M), dtype = A.dtype) - B = A.copy() - for i in range(rank): - maxi, maxj = divmod(abs(B).argmax(), M) - max_val = B[maxi, maxj] - U[:,i] = B[:,maxj].copy() - V[i] = B[maxi]/max_val - B -= U[:,i].reshape(-1, 1)*V[i].reshape(1, -1) - return U, V - -def py_lu_partial_fixed(A, rank): - N, M = A.shape - U = np.ndarray((N, rank), dtype = A.dtype) - V = np.ndarray((rank, M), dtype = A.dtype) - P = [i for i in range(N)] - maxi = 0 - i = 0 - index_in_P = 0 - while i < rank: - B = A[maxi]-U[maxi,:i].dot(V[:i]) - maxj = abs(B).argmax() - max_val = B[maxj] - if max_val == 0: - if len(P) == 1: - break - P.pop(index_in_P) - - else: - U[:,i] = A[:,maxj]-U[:,:i].dot(V[:i,maxj]) - V[i] = B/max_val - P.pop(index_in_P) - index_in_P = abs(U[P,i]).argmax() - maxi = P[index_in_P] - i += 1 - return U, V - -def aca_partial(A, tol): - N, M = A.shape - rank = 10 - U = np.ndarray((N, rank), dtype = A.dtype) - V = np.ndarray((rank, M), dtype = A.dtype) - P = [i for i in range(N)] - maxi = 0 - i = 0 - index_in_P = 0 - eps = 1+tol - while tol < eps: - B = A[maxi]-U[maxi,:i].dot(V[:i]) - maxj = abs(B).argmax() - max_val = B[maxj] - if max_val == 0: - if len(P) == 1: - break - else: - continue - U[:,i] = A[:,maxj]-U[:,:i].dot(V[:i,maxj]) - V[i] = B/max_val - P.pop(index_in_P) - index_in_P = abs(U[P,i]).argmax() - maxi = P[index_in_P] - if i == 0: - norm = np.linalg.norm(U[:,0])*np.linalg.norm(V[0]) - #print(norm) - eps = np.linalg.norm(U[:,i])*np.linalg.norm(V[i])/norm - #print(i, max_val, eps) - if i+1 == rank: - new_rank = min(2*rank, M, N) - tmp_U = np.ndarray((N, new_rank), dtype = A.dtype) - tmp_V = np.ndarray((new_rank, M), dtype = A.dtype) - tmp_U[:,:rank] = U - tmp_V[:rank] = V - U = tmp_U - V = tmp_V - rank = new_rank - i += 1 - return U[:,:i].copy(), V[:i].copy() - -def py_ica(A, tol): - N, M = A.shape - tol2 = tol*tol - max_rank = 100 - U = np.ndarray((N, max_rank), dtype = A.dtype) - V = np.ndarray((max_rank, M), dtype = A.dtype) - row_index = np.random.randint(N) - row = A[row_index] - rank = 0 - cross_norm = 0. - skeleton_norm = 0. - K = min(N, M) - while rank < K and skeleton_norm >= tol*cross_norm: - col_index = abs(row).argmax() - col = A[:, col_index] - U[:, :rank].dot(V[:rank, col_index]) - row_index = abs(col).argmax() - max_value = col[row_index] - if max_value == 0: - break - #print(row_index, col_index, max_value) - row = (A[row_index] - U[row_index, :rank].dot(V[:rank])) / max_value - skeleton_norm = np.linalg.norm(col) * np.linalg.norm(row) - skeleton_norm *= skeleton_norm - cross_norm = cross_norm+skeleton_norm+2*(col.conj().dot(U[:, :rank]).dot(V[:rank].dot(row.conj()))).real - #print(skeleton_norm, cross_norm) - if max_rank == rank: - max_rank = min(2*max_rank, K) - tmp_U = np.ndarray((N, max_rank), dtype = A.dtype) - tmp_V = np.ndarray((max_rank, M), dtype = A.dtype) - tmp_U[:,:rank] = U - tmp_V[:rank] = V - U, V = tmp_U, tmp_V - U[:,rank] = col.copy() - V[rank] = row.copy() - row[col_index] = 0 - rank += 1 - return U[:,:rank].copy(), V[:rank].copy() diff --git a/lib/maxvolpy/maxvolpy/maxvol.py b/lib/maxvolpy/maxvolpy/maxvol.py deleted file mode 100644 index ba4adc6..0000000 --- a/lib/maxvolpy/maxvolpy/maxvol.py +++ /dev/null @@ -1,678 +0,0 @@ -""" -This submodule contains routines to find good submatrices. How good -matrix is depends on special extreme properties of the matrix. Two of -this properties are 1-volume and 2-volume with the following formulas: - -:math:`vol_1(A) = \\left|\\det(A)\\right|,\\, vol_2(A) =\ - \\sqrt{\\max(\\det(A^HA), \\det(AA^H))}` -""" - -from __future__ import absolute_import, division, print_function - -__all__ = ['rect_maxvol', 'maxvol', 'rect_maxvol_svd', 'maxvol_svd', - 'rect_maxvol_qr', 'maxvol_qr'] - -from .misc import svd_cut - -def py_rect_maxvol(A, tol=1., maxK=None, min_add_K=None, minK=None, - start_maxvol_iters=10, identity_submatrix=True, top_k_index=-1, verbose=False): - """ - Python implementation of rectangular 2-volume maximization. - - See Also - -------- - rect_maxvol - """ - # tol2 - square of parameter tol - tol2 = tol**2 - # N - number of rows, r - number of columns of matrix A - N, r = A.shape - # some work on parameters - if N <= r: - return np.arange(N, dtype=np.int32), np.eye(N, dtype=A.dtype) - if maxK is None or maxK > N: - maxK = N - if maxK < r: - maxK = r - if minK is None or minK < r: - minK = r - if minK > N: - minK = N - if min_add_K is not None: - minK = max(minK, r + min_add_K) - if minK > maxK: - minK = maxK - #raise ValueError('minK value cannot be greater than maxK value') - if top_k_index == -1 or top_k_index > N: - top_k_index = N - if top_k_index < r: - top_k_index = r - # choose initial submatrix and coefficients according to maxvol - # algorithm - index = np.zeros(N, dtype=np.int32) - chosen = np.ones(top_k_index) - tmp_index, C = py_maxvol(A, 1.05, start_maxvol_iters, top_k_index) - index[:r] = tmp_index - chosen[tmp_index] = 0 - C = np.asfortranarray(C) - # compute square 2-norms of each row in coefficients matrix C - row_norm_sqr = np.array([chosen[i]*np.linalg.norm(C[i], 2)**2 for - i in range(top_k_index)]) - # find maximum value in row_norm_sqr - i = np.argmax(row_norm_sqr) - K = r - # set cgeru or zgeru for complex numbers and dger or sger - # for float numbers - try: - ger = get_blas_funcs('geru', [C]) - except: - ger = get_blas_funcs('ger', [C]) - # augment maxvol submatrix with each iteration - while (row_norm_sqr[i] > tol2 and K < maxK) or K < minK: - # add i to index and recompute C and square norms of each row - # by SVM-formula - index[K] = i - chosen[i] = 0 - c = C[i].copy() - v = C.dot(c.conj()) - l = 1.0/(1+v[i]) - ger(-l,v,c,a=C,overwrite_a=1) - C = np.hstack([C, l*v.reshape(-1,1)]) - row_norm_sqr -= (l*v[:top_k_index]*v[:top_k_index].conj()).real - row_norm_sqr *= chosen - # find maximum value in row_norm_sqr - i = row_norm_sqr.argmax() - K += 1 - # parameter identity_submatrix is True, set submatrix, - # corresponding to maxvol rows, equal to identity matrix - if identity_submatrix: - C[index[:K]] = np.eye(K, dtype=C.dtype) - return index[:K].copy(), C - -def py_maxvol(A, tol=1.05, max_iters=100, top_k_index=-1, verbose=False): - """ - Python implementation of 1-volume maximization. - - See Also - -------- - maxvol - """ - # some work on parameters - if tol < 1: - tol = 1.0 - N, r = A.shape - if N <= r: - return np.arange(N, dtype=np.int32), np.eye(N, dtype=A.dtype) - if top_k_index == -1 or top_k_index > N: - top_k_index = N - if top_k_index < r: - top_k_index = r - # set auxiliary matrices and get corresponding *GETRF function - # from lapack - B = np.copy(A[:top_k_index], order='F') - C = np.copy(A.T, order='F') - H, ipiv, info = get_lapack_funcs('getrf', [B])(B, overwrite_a=1) - # compute pivots from ipiv (result of *GETRF) - index = np.arange(N, dtype=np.int32) - for i in range(r): - tmp = index[i] - index[i] = index[ipiv[i]] - index[ipiv[i]] = tmp - # solve A = CH, H is in LU format - B = H[:r] - # It will be much faster to use *TRSM instead of *TRTRS - trtrs = get_lapack_funcs('trtrs', [B]) - trtrs(B, C, trans=1, lower=0, unitdiag=0, overwrite_b=1) - trtrs(B, C, trans=1, lower=1, unitdiag=1, overwrite_b=1) - # C has shape (r, N) -- it is stored transposed - # find max value in C - i, j = divmod(abs(C[:,:top_k_index]).argmax(), top_k_index) - # set cgeru or zgeru for complex numbers and dger or sger for - # float numbers - try: - ger = get_blas_funcs('geru', [C]) - except: - ger = get_blas_funcs('ger', [C]) - # set number of iters to 0 - iters = 0 - # check if need to swap rows - while abs(C[i,j]) > tol and iters < max_iters: - # add j to index and recompute C by SVM-formula - index[i] = j - tmp_row = C[i].copy() - tmp_column = C[:,j].copy() - tmp_column[i] -= 1. - alpha = -1./C[i,j] - ger(alpha, tmp_column, tmp_row, a=C, overwrite_a=1) - iters += 1 - i, j = divmod(abs(C[:,:top_k_index]).argmax(), top_k_index) - return index[:r].copy(), C.T - -def rect_maxvol(A, tol=1., maxK=None, min_add_K=None, minK=None, - start_maxvol_iters=10, identity_submatrix=True, top_k_index=-1): - """ - Finds good rectangular submatrix. - - Uses greedy iterative maximization of 2-volume to find good - `K`-by-`r` submatrix in a given `N`-by-`r` matrix `A` of rank `r`. - Returns good submatrix and least squares coefficients of expansion - (`N`-by-`K` matrix) of rows of matrix `A` by rows of good submatrix. - - Parameters - ---------- - A : numpy.ndarray(ndim=2) - Real or complex matrix of shape `(N, r)`, `N >= r`. - tol : float, optional - Upper bound for euclidian norm of coefficients of expansion of - rows of `A` by rows of good submatrix. Defaults to `1.0`. - maxK : integer, optional - Maximum number of rows in good submatrix. Defaults to `N` if - not set explicitly. - minK : integer, optional - Minimum number of rows in good submatrix. Defaults to `r` if - not set explicitly. - min_add_K : integer, optional - Minimum number of rows to add to the square submatrix. - Resulting good matrix will have minimum of `r+min_add_K` rows. - Ignored if not set explicitly. - start_maxvol_iters : integer, optional - How many iterations of square maxvol (optimization of 1-volume) - is required to be done before actual rectangular 2-volume - maximization. Defaults to `10`. - identity_submatrix : boolean, optional - Coefficients of expansions are computed as least squares - solution. If `identity_submatrix` is True, returned matrix of - coefficients will have submatrix, corresponding to good rows, - set to identity. Defaults to `True`. - top_k_index : integer, optional - Pivot rows for good submatrix will be in range from `0` to - `(top_k_index-1)`. This restriction is ignored, if `top_k_index` - is -1. Defaults to `-1`. - - Returns - ------- - piv : numpy.ndarray(ndim=1, dtype=numpy.int32) - Rows of matrix `A`, corresponding to submatrix, good in terms - of 2-volume. Shape is `(K, )`. - C : numpy.ndarray(ndim=2) - Matrix of coefficients of expansions of all rows of `A` by good - rows `piv`. Shape is `(N, K)`. - - Examples - -------- - >>> import numpy as np - >>> from maxvolpy.maxvol import rect_maxvol - >>> np.random.seed(100) - >>> a = np.random.rand(1000, 30, 2).view(dtype=np.complex128)[:,:,0] - >>> piv, C = rect_maxvol(a, 1.0) - >>> np.allclose(a, C.dot(a[piv])) - True - >>> print('maximum euclidian norm of row in matrix C: {:.5f}'. - ... format(max([np.linalg.norm(C[i], 2) for i in range(1000)]))) - maximum euclidian norm of row in matrix C: 1.00000 - >>> piv, C = rect_maxvol(a, 1.5) - >>> np.allclose(a, C.dot(a[piv])) - True - >>> print('maximum euclidian norm of row in matrix C: {:.5f}'. - ... format(max([np.linalg.norm(C[i], 2) for i in range(1000)]))) - maximum euclidian norm of row in matrix C: 1.49193 - >>> piv, C = rect_maxvol(a, 2.0) - >>> np.allclose(a, C.dot(a[piv])) - True - >>> print('maximum euclidian norm of row in matrix C: {:.5f}'. - ... format(max([np.linalg.norm(C[i], 2) for i in range(1000)]))) - maximum euclidian norm of row in matrix C: 1.91954 - """ - return rect_maxvol_func(A, tol, maxK, min_add_K, minK, start_maxvol_iters, - identity_submatrix, top_k_index) - -def maxvol(A, tol=1.05, max_iters=100, top_k_index=-1, verbose=False): - """ - Finds good square submatrix. - - Uses greedy iterative maximization of 1-volume to find good - `r`-by-`r` submatrix in a given `N`-by-`r` matrix `A` of rank `r`. - Returns good submatrix and coefficients of expansion - (`N`-by-`r` matrix) of rows of matrix `A` by rows of good submatrix. - - Parameters - ---------- - A : numpy.ndarray(ndim=2) - Real or complex matrix of shape `(N, r)`, `N >= r`. - tol : float, optional - Upper bound for infinite norm of coefficients of expansion of - rows of `A` by rows of good submatrix. Minimum value is 1. - Default to `1.05`. - max_iters : integer, optional - Maximum number of iterations. Each iteration swaps 2 rows. - Defaults to `100`. - top_k_index : integer, optional - Pivot rows for good submatrix will be in range from `0` to - `(top_k_index-1)`. This restriction is ignored, if `top_k_index` - is -1. Defaults to `-1`. - - Returns - ------- - piv : numpy.ndarray(ndim=1, dtype=numpy.int32) - Rows of matrix `A`, corresponding to submatrix, good in terms - of 1-volume. Shape is `(r, )`. - C : numpy.ndarray(ndim=2) - Matrix of coefficients of expansions of all rows of `A` by good - rows `piv`. Shape is `(N, r)`. - - Examples - -------- - >>> import numpy as np - >>> from maxvolpy.maxvol import maxvol - >>> np.random.seed(100) - >>> a = np.random.rand(1000, 30, 2).view(dtype=np.complex128)[:,:,0] - >>> piv, C = maxvol(a, 1.0) - >>> np.allclose(a, C.dot(a[piv])) - True - >>> print('Chebyshev norm of matrix C: {:.5f}'.format(abs(C).max())) - Chebyshev norm of matrix C: 1.00000 - >>> piv, C = maxvol(a, 1.05) - >>> np.allclose(a, C.dot(a[piv])) - True - >>> print('Chebyshev norm of matrix C: {:.5f}'.format(abs(C).max())) - Chebyshev norm of matrix C: 1.04641 - >>> piv, C = maxvol(a, 1.10) - >>> np.allclose(a, C.dot(a[piv])) - True - >>> print('Chebyshev norm of matrix C: {:.5f}'.format(abs(C).max())) - Chebyshev norm of matrix C: 1.07854 - """ - return maxvol_func(A, tol=tol, max_iters=max_iters, - top_k_index=top_k_index, verbose=verbose) - -def rect_maxvol_svd(A, svd_tol=1e-3, svd_alpha=0., tol=1., maxK=None, - min_add_K=None, minK=None, start_maxvol_iters=10, - identity_submatrix=True, job='F', top_k_index=-1): - """ - Applies SVD truncation and finds good rectangular submatrix. - - Computes SVD for `top_k_index` rows and/or columns of given matrix - `A`, increases singular values by regularizing parameter - `svd_alpha`, cuts off singular values, lower than - `svd_tol` (relatively, getting only highest singular vectors), - projects rows and/or columns, starting from `top_k_index`, to space - of first `top_k_index` rows and/or columns and runs `rect_maxvol` - for left and/or right singular vectors. - - Parameters - ---------- - A : numpy.ndarray(ndim=2) - Real or complex matrix. - svd_tol : float - Cut-off singular values parameter. - svd_alpha : float - Regularizing parameter for `misc.svd_cut`. - tol : float - Upper bound for euclidian norm of coefficients of expansion of - rows/columns of approximant of `A` by good rows/columns of - approximant. - maxK : integer - Maximum number of rows/columns in good submatrix. - minK : integer - Minimum number of rows/columns in good submatrix. - min_add_K : integer - Minimum number of rows/columns to add to the square submatrix. - Resulting good matrix will have minimum of `r+min_add_K` - rows/columns. - start_maxvol_iters : integer - How many iterations of square maxvol (optimization of 1-volume) - is required to be done before actual rectangular 2-volume - maximization. - identity_submatrix : boolean - Coefficients of expansions are computed as least squares - solution. If `identity_submatrix` is True, returned matrix of - coefficients will have submatrix, corresponding to good - rows/columns, set to identity. - job : character - 'R' to find good rows in approximant, 'C' to find good columns - in approximant and 'F' for both rows and columns. - top_k_index : integer - Pivot rows/columns for good submatrix will be in range from `0` - to `(top_k_index-1)`. This restriction is ignored, if - `top_k_index` is -1. - - Returns - ------- - Depending on `job` parameter, returns result of `rect_maxvol` for\ - left and/or right singular vectors of approximant. - piv : numpy.ndarray(ndim=1, dtype=numpy.int32) - Rows/columns of approximant of `A`, corresponding to submatrix, - good in terms of 2-volume. - C : numpy.ndarray(ndim=2) - Coefficients of expansions of all rows/columns of approximant - by good rows/columns `piv` of approximant. - - Examples - -------- - >>> import numpy as np - >>> from maxvolpy.maxvol import rect_maxvol_svd - >>> np.random.seed(100) - >>> a = np.random.rand(1000, 30, 2).view(dtype=np.complex128)[:,:,0] - >>> piv, C = rect_maxvol_svd(a, svd_tol=1e-1, tol=1.0, job='R') - >>> print('relative maxvol approximation error: {:.5f}'.format( - ... np.linalg.norm(a-C.dot(a[piv]), 2)/np.linalg.norm(a, 2))) - relative maxvol approximation error: 0.14497 - >>> print('maximum euclidian norm of row in matrix C: {:.5f}'. - ... format(max([np.linalg.norm(C[i], 2) for i in range(1000)]))) - maximum euclidian norm of row in matrix C: 1.00000 - >>> piv, C = rect_maxvol_svd(a, svd_tol=1e-1, tol=1.5, job='R') - >>> print('relative maxvol approximation error: {:.5f}'.format( - ... np.linalg.norm(a-C.dot(a[piv]), 2)/np.linalg.norm(a, 2))) - relative maxvol approximation error: 0.19640 - >>> print('maximum euclidian norm of row in matrix C: {:.5f}'. - ... format(max([np.linalg.norm(C[i], 2) for i in range(1000)]))) - maximum euclidian norm of row in matrix C: 1.49535 - >>> piv, C = rect_maxvol_svd(a, svd_tol=1e-1, tol=2.0, job='R') - >>> print('relative maxvol approximation error: {:.5f}'.format( - ... np.linalg.norm(a-C.dot(a[piv]), 2)/np.linalg.norm(a, 2))) - relative maxvol approximation error: 0.22981 - >>> print('maximum euclidian norm of row in matrix C: {:.5f}'. - ... format(max([np.linalg.norm(C[i], 2) for i in range(1000)]))) - maximum euclidian norm of row in matrix C: 1.88485 - """ - if job == 'R': - if top_k_index == -1: - top_k_index = A.shape[0] - # compute largest singular values and vectors for first - # top_k_index rows - U, S, V = svd_cut(A[:top_k_index], svd_tol, svd_alpha) - # find projection coefficients of all other rows to subspace - # of largest singular vectors of first rows - B = A[top_k_index:].dot(V.T.conj())*(1.0/S).reshape(1,-1) - # apply rect_maxvol for projection coefficients - return rect_maxvol(np.vstack([U, B]), tol, maxK, min_add_K, minK, - start_maxvol_iters, identity_submatrix, top_k_index) - elif job == 'C': - if top_k_index == -1: - top_k_index = A.shape[1] - # compute largest singular values and vectors for first - # top_k_index columns - U, S, V = svd_cut(A[:,:top_k_index], svd_tol, svd_alpha) - # find projection coefficients of all other columns to subspace - # of largest singular vectors of first columns - B = (1.0/S).reshape(-1,1)*U.T.conj().dot(A[:,top_k_index:]) - # apply rect_maxvol for projection coefficients - return rect_maxvol(np.vstack([V.T.conj(), B.T.conj()]), tol, maxK, - min_add_K, minK, start_maxvol_iters, identity_submatrix, - top_k_index) - elif job == 'F': - # proceed with job = 'R' and job = 'C' simultaneously - if top_k_index != -1: - value0, value1 = rect_maxvol_svd(A, svd_tol, svd_alpha, tol, maxK, - min_add_K, minK, start_maxvol_iters, identity_submatrix, - 'R', top_k_index) - value2, value3 = rect_maxvol_svd(A, svd_tol, svd_alpha, tol, maxK, - min_add_K, minK, start_maxvol_iters, identity_submatrix, - 'C', top_k_index) - return value0, value1, value2, value3 - U, S, V = svd_cut(A, svd_tol, svd_alpha) - value0, value1 = rect_maxvol(U, tol, maxK, min_add_K, minK, - start_maxvol_iters, identity_submatrix, top_k_index) - value2, value3 = rect_maxvol(V.T.conj(), tol, maxK, min_add_K, minK, - start_maxvol_iters, identity_submatrix, top_k_index) - return value0, value1, value2, value3 - -def maxvol_svd(A, svd_tol=1e-3, svd_alpha=0., tol=1.05, max_iters=100, job='F', - top_k_index=-1): - """ - Applies SVD truncation and finds good square submatrix. - - Computes SVD for `top_k_index` rows and/or columns of given matrix - `A`, increases singular values by regularizing parameter - `svd_alpha`, cuts off singular values, lower than - `svd_tol` (relatively, getting only highest singular vectors), - projects rows and/or columns, starting from `top_k_index`, to space - of first `top_k_index` rows and/or columns and runs `maxvol` - for left and/or right singular vectors. - - Parameters - ---------- - A : numpy.ndarray(ndim=2) - Real or complex matrix. - svd_tol : float - Cut-off singular values parameter. - svd_alpha : float - Regularizing parameter for `misc.svd_cut`. - tol : float - Upper bound for infinite norm of coefficients of expansion of - rows/columns of approximant of `A` good rows/columns of - approximant. Minimum value is 1. - max_iters : integer - Maximum number of iterations. Each iteration swaps 2 rows. - job : character - 'R' to find good rows in approximant, 'C' to find good columns - in approximant and 'F' for both rows and columns. - top_k_index : integer - Pivot rows/columns for good submatrix will be in range from `0` - to `(top_k_index-1)`. This restriction is ignored, if - `top_k_index` is -1. - - Returns - ------- - Depending on `job` parameter, returns result of `rect_maxvol` for\ - left and/or right singular vectors of approximant. - piv : numpy.ndarray(ndim=1, dtype=numpy.int32) - Rows/columns of approximant of `A`, corresponding to submatrix, - good in terms of 1-volume. - C : numpy.ndarray(ndim=2) - Coefficients of expansions of all rows/columns of approximant - by good rows/columns `piv` of approximant. - - Examples - -------- - >>> import numpy as np - >>> from maxvolpy.maxvol import maxvol_svd - >>> np.random.seed(100) - >>> a = np.random.rand(1000, 30, 2).view(dtype=np.complex128)[:,:,0] - >>> piv, C = maxvol_svd(a, svd_tol=1e-1, tol=1.0, job='R') - >>> print('relative maxvol approximation error: {:.5f}'.format( - ... np.linalg.norm(a-C.dot(a[piv]), 2)/np.linalg.norm(a, 2))) - relative maxvol approximation error: 0.24684 - >>> print('Chebyshev norm of matrix C: {:.5f}'.format(abs(C).max())) - Chebyshev norm of matrix C: 1.00000 - >>> piv, C = maxvol_svd(a, svd_tol=1e-1, tol=1.05, job='R') - >>> print('relative maxvol approximation error: {:.5f}'.format( - ... np.linalg.norm(a-C.dot(a[piv]), 2)/np.linalg.norm(a, 2))) - relative maxvol approximation error: 0.24684 - >>> print('Chebyshev norm of matrix C: {:.5f}'.format(abs(C).max())) - Chebyshev norm of matrix C: 1.00000 - >>> piv, C = maxvol_svd(a, svd_tol=1e-1, tol=1.10, job='R') - >>> print('relative maxvol approximation error: {:.5f}'.format( - ... np.linalg.norm(a-C.dot(a[piv]), 2)/np.linalg.norm(a, 2))) - relative maxvol approximation error: 0.25485 - >>> print('Chebyshev norm of matrix C: {:.5f}'.format(abs(C).max())) - Chebyshev norm of matrix C: 1.06825 - """ - if tol < 1: - tol = 1.0 - if job == 'R': - if top_k_index == -1: - top_k_index = A.shape[0] - # compute largest singular values and vectors for first - # top_k_index rows - U, S, V = svd_cut(A[:top_k_index], svd_tol, svd_alpha) - # find projection coefficients of all other rows to subspace of - # largest singular vectors of first rows - B = A[top_k_index:].dot(V.T.conj())*(1.0/S).reshape(1,-1) - # apply maxvol for projection coefficients - return maxvol(np.vstack([U, B]), tol, max_iters, top_k_index) - elif job == 'C': - if top_k_index == -1: - top_k_index = A.shape[1] - # compute largest singular values and vectors for first - # top_k_index columns - U, S, V = svd_cut(A[:,:top_k_index], svd_tol, svd_alpha) - # find projection coefficients of all other columns to subspace - # of largest singular vectors of first columns - B = (1.0/S).reshape(-1,1)*U.T.conj().dot(A[:,top_k_index:]) - # apply rect_maxvol for projection coefficients - return maxvol(np.vstack([V.T.conj(), B.T.conj()]), tol, max_iters, - top_k_index) - elif job == 'F': - # procede with job = 'R' and job = 'C' simultaneously - if top_k_index != -1: - value0, value1 = maxvol_svd(A, svd_tol, svd_alpha, tol, max_iters, - 'R', top_k_index) - value2, value3 = maxvol_svd(A, svd_tol, svd_alpha, tol, max_iters, - 'C', top_k_index) - return value0, value1, value2, value3 - U, S, V = svd_cut(A, svd_tol, svd_alpha) - value0, value1 = maxvol(U, tol, max_iters, top_k_index) - value2, value3 = maxvol(V.T.conj(), tol, max_iters, top_k_index) - return value0, value1, value2, value3 - -def rect_maxvol_qr(A, tol=1., maxK=None, min_add_K=None, minK=None, - start_maxvol_iters=10, identity_submatrix=True, top_k_index=-1): - """ - Finds good rectangular submatrix in Q factor of QR of `A`. - - When rank of `N`-by-`r` matrix `A` is not guaranteed to be - equal to `r`, good submatrix in `A` can be found as good submatrix - in Q factor of QR decomposition of `A`. - - Parameters - ---------- - A : numpy.ndarray(ndim=2) - Real or complex matrix of shape `(N, r)`, `N >= r`. - tol : float - Upper bound for euclidian norm of coefficients of expansion of - rows of `A` by rows of good submatrix. - maxK : integer - Maximum number of rows in good submatrix. - minK : integer - Minimum number of rows in good submatrix. - min_add_K : integer - Minimum number of rows to add to the square submatrix. - Resulting good matrix will have minimum of `r+min_add_K` rows. - start_maxvol_iters : integer - How many iterations of square maxvol (optimization of 1-volume) - is required to be done before actual rectangular 2-volume - maximization. - identity_submatrix : boolean - Coefficients of expansions are computed as least squares - solution. If `identity_submatrix` is True, returned matrix of - coefficients will have submatrix, corresponding to good rows, - set to identity. - top_k_index : integer - Pivot rows for good submatrix will be in range from `0` to - `(top_k_index-1)`. This restriction is ignored, if `top_k_index` - is -1. - - Returns - ------- - piv : numpy.ndarray(ndim=1, dtype=numpy.int32) - Rows of matrix `A`, corresponding to submatrix, good in terms - of 2-volume. Shape is `(K, )`. - C : numpy.ndarray(ndim=2) - Matrix of coefficients of expansions of all rows of `A` by good - rows `piv`. Shape is `(N, K)`. - - Examples - -------- - >>> import numpy as np - >>> from maxvolpy.maxvol import rect_maxvol_qr - >>> np.random.seed(100) - >>> a = np.random.rand(1000, 30, 2).view(dtype=np.complex128)[:,:,0] - >>> piv, C = rect_maxvol_qr(a, 1.0) - >>> np.allclose(a, C.dot(a[piv])) - True - >>> print('maximum euclidian norm of row in matrix C: {:.5f}'. - ... format(max([np.linalg.norm(C[i], 2) for i in range(1000)]))) - maximum euclidian norm of row in matrix C: 1.00000 - >>> piv, C = rect_maxvol_qr(a, 1.5) - >>> np.allclose(a, C.dot(a[piv])) - True - >>> print('maximum euclidian norm of row in matrix C: {:.5f}'. - ... format(max([np.linalg.norm(C[i], 2) for i in range(1000)]))) - maximum euclidian norm of row in matrix C: 1.49193 - >>> piv, C = rect_maxvol_qr(a, 2.0) - >>> np.allclose(a, C.dot(a[piv])) - True - >>> print('maximum euclidian norm of row in matrix C: {:.5f}'. - ... format(max([np.linalg.norm(C[i], 2) for i in range(1000)]))) - maximum euclidian norm of row in matrix C: 1.91954 - """ - N, r = A.shape - if N <= r: - return np.arange(N, dtype=np.int32), np.eye(N, dtype=A.dtype) - Q = np.linalg.qr(A)[0] - return rect_maxvol(Q, tol, maxK, min_add_K, minK, start_maxvol_iters, - identity_submatrix, top_k_index) - -def maxvol_qr(A, tol=1.05, max_iters=100, top_k_index=-1): - """ - Finds good square submatrix in Q factor of QR of `A`. - - When rank of `N`-by-`r` matrix `A` is not guaranteed to be - equal to `r`, good submatrix in `A` can be found as good submatrix - in Q factor of QR decomposition of `A`. - - Parameters - ---------- - A : numpy.ndarray(ndim=2) - Real or complex matrix of shape `(N, r)`, `N >= r`. - tol : float - Upper bound for infinite norm of coefficients of expansion of rows - of `A` by rows of good submatrix. Minimum value is 1. - max_iters : integer - Maximum number of iterations. Each iteration swaps 2 rows. - top_k_index : integer - Pivot rows for good submatrix will be in range from `0` to - `(top_k_index-1)`. This restriction is ignored, if `top_k_index` - is -1. - - Returns - ------- - piv : numpy.ndarray(ndim=1, dtype=numpy.int32) - Rows of matrix `A`, corresponding to submatrix, good in terms - of 1-volume. Shape is `(r, )`. - C : numpy.ndarray(ndim=2) - Matrix of coefficients of expansions of all rows of `A` by good - rows `piv`. Shape is `(N, r)`. - - Examples - -------- - >>> import numpy as np - >>> from maxvolpy.maxvol import maxvol_qr - >>> np.random.seed(100) - >>> a = np.random.rand(1000, 30, 2).view(dtype=np.complex128)[:,:,0] - >>> piv, C = maxvol_qr(a, 1.0) - >>> np.allclose(a, C.dot(a[piv])) - True - >>> print('Chebyshev norm of matrix C: {:.5f}'.format(abs(C).max())) - Chebyshev norm of matrix C: 1.00000 - >>> piv, C = maxvol_qr(a, 1.05) - >>> np.allclose(a, C.dot(a[piv])) - True - >>> print('Chebyshev norm of matrix C: {:.5f}'.format(abs(C).max())) - Chebyshev norm of matrix C: 1.04641 - >>> piv, C = maxvol_qr(a, 1.10) - >>> np.allclose(a, C.dot(a[piv])) - True - >>> print('Chebyshev norm of matrix C: {:.5f}'.format(abs(C).max())) - Chebyshev norm of matrix C: 1.07854 - """ - N, r = A.shape - if N <= r: - return np.arange(N, dtype=np.int32), np.eye(N, dtype=A.dtype) - Q = np.linalg.qr(A)[0] - return maxvol(Q, tol, max_iters, top_k_index) - -import numpy as np - -try: - from ._maxvol import c_rect_maxvol, c_maxvol - rect_maxvol_func = c_rect_maxvol - maxvol_func = c_maxvol - __all__.extend(['c_rect_maxvol', 'c_maxvol']) -except: - from scipy.linalg import solve_triangular, get_lapack_funcs, get_blas_funcs - print("warning: fast C maxvol functions are not compiled, continue with" - " python maxvol functions") - rect_maxvol_func = py_rect_maxvol - maxvol_func = py_maxvol diff --git a/lib/maxvolpy/maxvolpy/misc.py b/lib/maxvolpy/maxvolpy/misc.py deleted file mode 100644 index 1387696..0000000 --- a/lib/maxvolpy/maxvolpy/misc.py +++ /dev/null @@ -1,84 +0,0 @@ -""" -:py:mod:`maxvolpy.misc` module contains some auxiliary routines, -useful when working with different maxvol functions from -:py:mod:`maxvolpy.maxvol` and when building approximations with help -of functions from :py:mod:`maxvolpy.cross`. -""" - -from __future__ import print_function, division, absolute_import - -__all__ = ['svd_cut', 'reduce_approximation'] - -import numpy as np - -def svd_cut(A, tol, alpha=0., norm=2): - """ - Computes SVD and cuts low singular values. - - Computes singular values decomposition of matrix `A`, adds - regularizing parameter `alpha` to each singular value and returns - only largest singular values and vectors with relative tolerance - `tol`. - - Parameters - ---------- - A: numpy.ndarray - Real or complex matrix or matrix-like object. - tol: float - Tolerance of cutting singular values operation. - alpha: float, optional - Regularizing parameter. - norm: {2, 'fro'}, optional - Defines norm, that is chosen when cutting singular values. - - Returns - ------- - U: numpy.ndarray - Left singular vectors, corresponding to largest singular values. - S: numpy.ndarray - Largest singular values. - V: numpy.ndarray - Right singular vectors, corresponding to largest singular values. - """ - U, S, V = np.linalg.svd(A, full_matrices=0) - S_reg = S+alpha - S1 = S_reg[::-1] - if norm == 2: - tmp_S = tol*S1[-1] - rank = S1.shape[0]-S1.searchsorted(tmp_S, 'left') - elif norm == 'fro': - S1 = S1*S1 - for i in range(1, S1.shape[0]): - S1[i] += S1[i-1] - tmp_S = S1[-1]*tol*tol - rank = S1.shape[0]-S1.searchsorted(tmp_S, 'left') - else: - raise ValueError("Invalid parameter norm value") - return U[:,:rank].copy(), S_reg[:rank].copy(), V[:rank].copy() - -def reduce_approximation(U, V, tol, alpha=0.): - """ - Performs `svd_cut` procedure for matrix `U.dot(V)` - - Parameters - ---------- - U, V: numpy.ndarray - Two matrices, such that `U.dot(V)` makes sense. - tol: float - Relative tolerance of cutting singular values of a matrix - `U.dot(V)`. - alpha: float - Regularizing parameter for `svd_cut`. - - Returns - ------- - U: numpy.ndarray - Left singular vectors, corresponding to largest singular values. - S: numpy.ndarray - Largest singular values. - V: numpy.ndarray - Right singular vectors, corresponding to largest singular values. - """ - Q1, R1 = np.linalg.qr(U) - U1, S1, V1 = svd_cut(R1.dot(V), tol, alpha) - return Q1.dot(U1), S1, V1 diff --git a/lib/maxvolpy/maxvolpy/setup.py b/lib/maxvolpy/maxvolpy/setup.py deleted file mode 100644 index 4119fc1..0000000 --- a/lib/maxvolpy/maxvolpy/setup.py +++ /dev/null @@ -1,26 +0,0 @@ -def configuration(parent_package=None, top_path=None): - from numpy.distutils.misc_util import Configuration - import numpy.__config__ as npconf - import os - from os.path import exists, getmtime - config = Configuration('maxvolpy', parent_package, top_path) - - start_setup_dir = os.getcwd() - cur_setup_dir = os.path.dirname(__file__) - if len(cur_setup_dir) > 0: - os.chdir(cur_setup_dir) - if not exists('_maxvol.pyx') or \ - getmtime('_maxvol.pyx.src') > getmtime('_maxvol.pyx'): - exec(open('_maxvol.pyx.src').read()) - os.chdir(start_setup_dir) - - config.add_extension('_maxvol', - sources=['_maxvol.pyx'], - extra_info=npconf.lapack_opt_info - ) - return config - - -if __name__ == '__main__': - from numpy.distutils.core import setup - setup(**configuration().todict()) diff --git a/lib/maxvolpy/requirements.txt b/lib/maxvolpy/requirements.txt deleted file mode 100644 index cb29d8b..0000000 --- a/lib/maxvolpy/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -numpy>=1.10.1 -numpydoc -scipy -cython>=0.23.4 diff --git a/lib/maxvolpy/setup.cfg b/lib/maxvolpy/setup.cfg deleted file mode 100644 index 3c6e79c..0000000 --- a/lib/maxvolpy/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[bdist_wheel] -universal=1 diff --git a/lib/maxvolpy/setup.py b/lib/maxvolpy/setup.py deleted file mode 100644 index abb8d13..0000000 --- a/lib/maxvolpy/setup.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -""" -Cross and skeleton approximations, based on maximum volume submatrices. -""" - -DOCLINES = (__doc__ or '').split("\n") - -from setuptools import setup -from setuptools.extension import Extension -from Cython.Build import cythonize -import numpy -import platform - -extensions = [ - Extension("maxvolpy._maxvol", - ["maxvolpy/_maxvol.pyx"], - include_dirs=[numpy.get_include()], - extra_compile_args=['-O3', '-march=native', '-ffast-math'] - ) -] - -ext_modules = None -if platform.system() != "Windows": - ext_modules = cythonize(extensions) - -setup( - name='maxvolpy', - version='0.3.8', - maintainer="Alexander Mikhalev", - maintainer_email="muxasizhevsk@gmail.com", - description=DOCLINES[1], - long_description=DOCLINES[1], - url="https://bitbucket.org/muxas/maxvolpy", - author="Alexander Mikhalev", - author_email="muxasizhevsk@gmail.com", - license='MIT', - install_requires=['numpy>=1.10.1', 'scipy>=0.16.0', 'cython>=0.23.4'], - packages=['maxvolpy'], - ext_modules=ext_modules -) From 03660948308ac53542071bcb4bf84d8a49640e91 Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Fri, 4 Sep 2026 12:29:58 +0200 Subject: [PATCH 12/30] build: move metadata to pyproject.toml, drop versioneer, unpin numpy ceiling Metadata -------- name, version, authors, description, readme, license, classifiers, dependencies, package discovery, package data and scripts now live in pyproject.toml's [project]/[tool.setuptools] tables, matching how grace-tensorpotential declares itself. setup.py keeps only what setuptools has no declarative form for -- ext_modules and the CMakeBuild cmdclass -- and drops from 305 to ~250 lines. The bin/ scripts stay script-files rather than becoming [project.scripts] console entry points: they are standalone files with a shebang calling main(sys.argv[1:]), so converting them (which would also fix Windows) needs each one refactored into a module first. versioneer ---------- Removed: versioneer.py, src/pyace/_version.py, setup.cfg, MANIFEST.in and .version -- the last four existed only to serve it. Version is now stated once in pyproject.toml, and pyace.__version__ reads it back via importlib.metadata, falling back to "unknown" in an uninstalled source tree. Version set to 0.4.0, taken from the .version file this branch had already bumped. Note this is a visible change: versioneer derived it from git describe and was reporting 0.2.7+N.g, while the newest tag (0.2.8) is not an ancestor of this branch. 0.4.0 is what a source tarball would already have reported, and it is written into potential metadata as "pacemaker_version". numpy ----- Dropped the "<2.2.0" ceiling. It came from d0f7a0b ("adjust numpy<2.2.0 with tensorflow 2.19.1"), but pyace does not depend on tensorflow, and tensorflow 2.19.1 declares numpy<2.2.0 itself, so pip enforces it wherever it matters. grace-tensorpotential allows tensorflow<=2.20, and tensorflow 2.20.0 requires only numpy>=1.26.0 -- so keeping our own stale ceiling would block an otherwise valid grace environment. Keep numpy>=2.0, which d0f7a0b had dropped by replacing the whole line, and which a7e19fa deliberately introduced. Correction to b64e60e's message: the build-time numpy does not set the runtime floor. A binary built against the numpy 2.x C-API also runs under numpy 1.19+, so building against >=2.0 is the widest-compatibility choice, and the runtime floor is a decision about the Python code, not the ABI. Verified in tp2: version 0.4.0 reported by both metadata and `pacemaker -v`, all 7 pybind11 modules plus the maxvol kernel import, all 9 scripts installed, package data present, `pip check` clean alongside tensorpotential 0.6.1 / tensorflow 2.19.1, and 292 tests passing. --- .version | 1 - MANIFEST.in | 2 - pyproject.toml | 68 +- setup.cfg | 13 - setup.py | 60 +- src/pyace/__init__.py | 7 +- src/pyace/_version.py | 623 ------------ versioneer.py | 2082 ----------------------------------------- 8 files changed, 76 insertions(+), 2780 deletions(-) delete mode 100644 .version delete mode 100644 MANIFEST.in delete mode 100644 setup.cfg delete mode 100644 src/pyace/_version.py delete mode 100644 versioneer.py diff --git a/.version b/.version deleted file mode 100644 index 1d0ba9e..0000000 --- a/.version +++ /dev/null @@ -1 +0,0 @@ -0.4.0 diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 827f0b9..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,2 +0,0 @@ -include versioneer.py -include src/pyace/_version.py diff --git a/pyproject.toml b/pyproject.toml index 0d0d6be..1276e59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,10 +4,9 @@ # make, which schedules this project's seven modules much less effectively, and # without Cython the maxvol extension is skipped entirely. # -# numpy>=2.0 is a *build* pin, not a preference: pyace.maxvol._maxvol is a -# Cython extension against the numpy C-API, so the numpy it is compiled against -# sets the minimum numpy it will import under at runtime. Keep this consistent -# with install_requires in setup.py. +# Build against numpy>=2.0 for pyace.maxvol._maxvol: a binary compiled against +# the 2.x C-API also runs under numpy 1.19+, so this is the widest-compatibility +# choice rather than a floor imposed on users. requires = [ "setuptools>=64", "wheel", @@ -18,3 +17,64 @@ requires = [ "scipy", ] build-backend = "setuptools.build_meta" + +[project] +name = "pyace" +version = "0.4.0" +authors = [ + { name = "Yury Lysogorskiy", email = "yury.lysogorskiy@rub.de" }, + { name = "Anton Bochkarev" }, + { name = "Sarath Menon" }, + { name = "Ralf Drautz" }, +] +description = 'Python bindings, utilities for PACE and fitting code "pacemaker"' +readme = "README.md" +requires-python = ">=3.9" +license = { text = "Academic Software License (ASL)" } +classifiers = [ + "License :: Other/Proprietary License", + "Programming Language :: Python :: 3", +] +dependencies = [ + # No numpy upper bound on purpose. The old "<2.2.0" came from tensorflow + # 2.19.1, which declares numpy<2.2.0 itself -- but pyace does not depend on + # tensorflow, and grace-tensorpotential allows tensorflow<=2.20, whose + # ceiling is gone (2.20.0 requires only numpy>=1.26.0). Repeating a stale + # ceiling here would block an otherwise valid grace environment. + "numpy>=2.0", + "ase", + "pandas", + "ruamel.yaml", + "psutil", + "scikit-learn", + "scipy", +] + +[project.urls] +"Homepage" = "https://github.com/ICAMS/python-ace" + +[tool.setuptools] +zip-safe = false +# These are plain scripts with a shebang, not importable main() entry points, +# so they are installed as files. Converting them to [project.scripts] console +# entry points would also make them work on Windows, but needs each bin/ script +# refactored into a module first. +script-files = [ + "bin/pacemaker", + "bin/pace_yaml2yace", + "bin/pace_timing", + "bin/pace_info", + "bin/pace_activeset", + "bin/pace_select", + "bin/pace_collect", + "bin/pace_augment", + "bin/pace_corerep", +] + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +"pyace.data" = ["mus_ns_uni_to_rawlsLS_np_rank.pckl", "input_template.yaml"] +# MIT notice must travel with the vendored maxvol kernel +"pyace.maxvol" = ["LICENSE.maxvolpy.txt"] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 3f6b41c..0000000 --- a/setup.cfg +++ /dev/null @@ -1,13 +0,0 @@ - -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. - -[versioneer] -VCS = git -style = pep440 -versionfile_source = src/pyace/_version.py -versionfile_build = pyace/_version.py -tag_prefix = -parentdir_prefix = - diff --git a/setup.py b/setup.py index 8397cdd..ab2a626 100644 --- a/setup.py +++ b/setup.py @@ -6,14 +6,9 @@ import subprocess import sys from pathlib import Path -from setuptools import Extension, setup, find_packages +from setuptools import Extension, setup from setuptools.command.build_ext import build_ext -import versioneer - -with open('README.md') as readme_file: - readme = readme_file.read() - # Convert distutils Windows platform specifiers to CMake -A arguments PLAT_TO_CMAKE = { @@ -83,8 +78,8 @@ def default_parallel_jobs() -> int: memory = _available_memory() if memory: - jobs = max(1, min(jobs, memory // (2 * 1024 ** 3))) - return jobs + jobs = min(jobs, memory // (2 * 1024 ** 3)) + return max(1, jobs) def compiler_launcher() -> str | None: @@ -244,24 +239,10 @@ def maxvol_extension(): ], language_level=3) -# The information here can also be placed in setup.cfg - better separation of -# logic and declaration, and simpler if you include description/version in a file. setup( - name='pyace', - version=versioneer.get_version(), - author='Yury Lysogorskiy, Anton Bochkarev, Sarath Menon, Ralf Drautz', - author_email='yury.lysogorskiy@rub.de', - description='Python bindings, utilities for PACE and fitting code "pacemaker"', - long_description=readme, - long_description_content_type='text/markdown', - - # tell setuptools to look for any packages under 'src' - packages=find_packages('src'), - # tell setuptools that all packages will be under the 'src' directory - # and nowhere else - package_dir={'': 'src'}, - - # add an extension module named 'python_cpp_example' to the package + # Everything declarative -- name, version, dependencies, packages, scripts, + # package data -- lives in pyproject.toml. Only the pieces setuptools has + # no declarative form for stay here. ext_modules=[CMakeExtension('pyace/sharmonics'), CMakeExtension('pyace/coupling'), CMakeExtension('pyace/basis'), @@ -271,32 +252,5 @@ def maxvol_extension(): CMakeExtension('pyace/grace_fs'), *maxvol_extension(), ], - # add custom build_ext command - cmdclass=versioneer.get_cmdclass(dict(build_ext=CMakeBuild)), - zip_safe=False, - url='https://github.com/ICAMS/python-ace', - install_requires=['numpy>=2.0,<2.2.0', - 'ase', - 'pandas', - 'ruamel.yaml', - 'psutil', - 'scikit-learn', - 'scipy' - ], - classifiers=[ - 'Programming Language :: Python :: 3', - ], - package_data={"pyace.data": [ - "mus_ns_uni_to_rawlsLS_np_rank.pckl", - "input_template.yaml" - ], - # MIT notice must travel with the vendored maxvol kernel - "pyace.maxvol": ["LICENSE.maxvolpy.txt"], - }, - scripts=["bin/pacemaker", "bin/pace_yaml2yace", - "bin/pace_timing", "bin/pace_info", - "bin/pace_activeset", "bin/pace_select", - "bin/pace_collect", "bin/pace_augment", "bin/pace_corerep"], - - python_requires=">=3.8" + cmdclass={"build_ext": CMakeBuild}, ) diff --git a/src/pyace/__init__.py b/src/pyace/__init__.py index 2571de9..038b2eb 100644 --- a/src/pyace/__init__.py +++ b/src/pyace/__init__.py @@ -38,6 +38,9 @@ "create_multispecies_basis_config", ] -from . import _version +from importlib.metadata import PackageNotFoundError, version as _package_version -__version__ = _version.get_versions()['version'] +try: + __version__ = _package_version("pyace") +except PackageNotFoundError: # imported from a source tree that was never installed + __version__ = "unknown" diff --git a/src/pyace/_version.py b/src/pyace/_version.py deleted file mode 100644 index e5b92af..0000000 --- a/src/pyace/_version.py +++ /dev/null @@ -1,623 +0,0 @@ - -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.20 (https://github.com/python-versioneer/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "$Format:%d$" - git_full = "$Format:%H$" - git_date = "$Format:%ci$" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: # pylint: disable=too-few-public-methods - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "pep440" - cfg.tag_prefix = "" - cfg.parentdir_prefix = "" - cfg.versionfile_source = "src/pyace/_version.py" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Create decorator to mark a method as the handler of a VCS.""" - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - return decorate - - -# pylint:disable=too-many-arguments,consider-using-with # noqa -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - process = None - for command in commands: - try: - dispcmd = str([command] + args) - # remember shell=False, so use git.cmd on windows, not just git - process = subprocess.Popen([command] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None, None - stdout = process.communicate()[0].strip().decode() - if process.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, process.returncode - return stdout, process.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for _ in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print("Tried directories %s but none started with prefix %s" % - (str(rootdirs), parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - with open(versionfile_abs, "r") as fobj: - for line in fobj: - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if "refnames" not in keywords: - raise NotThisMethod("Short version file found") - date = keywords.get("date") - if date is not None: - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = {r.strip() for r in refnames.strip("()").split(",")} - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r'\d', r)} - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - # Filter out refs that exactly match prefix or that don't start - # with a number once the prefix is stripped (mostly a concern - # when prefix is '') - if not re.match(r'\d', r): - continue - if verbose: - print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = runner(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], - cwd=root) - # --abbrev-ref was added in git-1.6.3 - if rc != 0 or branch_name is None: - raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") - branch_name = branch_name.strip() - - if branch_name == "HEAD": - # If we aren't exactly on a branch, pick a branch which represents - # the current commit. If all else fails, we are on a branchless - # commit. - branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) - # --contains was added in git-1.5.4 - if rc != 0 or branches is None: - raise NotThisMethod("'git branch --contains' returned error") - branches = branches.split("\n") - - # Remove the first line if we're running detached - if "(" in branches[0]: - branches.pop(0) - - # Strip off the leading "* " from the list of branches. - branches = [branch[2:] for branch in branches] - if "master" in branches: - branch_name = "master" - elif not branches: - branch_name = None - else: - # Pick the first branch that is returned. Good or bad. - branch_name = branches[0] - - pieces["branch"] = branch_name - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = runner(GITS, ["rev-list", "HEAD", "--count"], cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_branch(pieces): - """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . - - The ".dev0" means not master branch. Note that .dev0 sorts backwards - (a feature branch will appear "older" than the master branch). - - Exceptions: - 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0" - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post0.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post0.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post0.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post0.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_post_branch(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . - - The ".dev0" means not master branch. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-branch": - rendered = render_pep440_branch(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-post-branch": - rendered = render_pep440_post_branch(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for _ in cfg.versionfile_source.split('/'): - root = os.path.dirname(root) - except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None} - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", "date": None} diff --git a/versioneer.py b/versioneer.py deleted file mode 100644 index ab0591d..0000000 --- a/versioneer.py +++ /dev/null @@ -1,2082 +0,0 @@ - -# Version: 0.20 - -"""The Versioneer - like a rocketeer, but for versions. - -The Versioneer -============== - -* like a rocketeer, but for versions! -* https://github.com/python-versioneer/python-versioneer -* Brian Warner -* License: Public Domain -* Compatible with: Python 3.6, 3.7, 3.8, 3.9 and pypy3 -* [![Latest Version][pypi-image]][pypi-url] -* [![Build Status][travis-image]][travis-url] - -This is a tool for managing a recorded version number in distutils-based -python projects. The goal is to remove the tedious and error-prone "update -the embedded version string" step from your release process. Making a new -release should be as easy as recording a new tag in your version-control -system, and maybe making new tarballs. - - -## Quick Install - -* `pip install versioneer` to somewhere in your $PATH -* add a `[versioneer]` section to your setup.cfg (see [Install](INSTALL.md)) -* run `versioneer install` in your source tree, commit the results -* Verify version information with `python setup.py version` - -## Version Identifiers - -Source trees come from a variety of places: - -* a version-control system checkout (mostly used by developers) -* a nightly tarball, produced by build automation -* a snapshot tarball, produced by a web-based VCS browser, like github's - "tarball from tag" feature -* a release tarball, produced by "setup.py sdist", distributed through PyPI - -Within each source tree, the version identifier (either a string or a number, -this tool is format-agnostic) can come from a variety of places: - -* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows - about recent "tags" and an absolute revision-id -* the name of the directory into which the tarball was unpacked -* an expanded VCS keyword ($Id$, etc) -* a `_version.py` created by some earlier build step - -For released software, the version identifier is closely related to a VCS -tag. Some projects use tag names that include more than just the version -string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool -needs to strip the tag prefix to extract the version identifier. For -unreleased software (between tags), the version identifier should provide -enough information to help developers recreate the same tree, while also -giving them an idea of roughly how old the tree is (after version 1.2, before -version 1.3). Many VCS systems can report a description that captures this, -for example `git describe --tags --dirty --always` reports things like -"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the -0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has -uncommitted changes). - -The version identifier is used for multiple purposes: - -* to allow the module to self-identify its version: `myproject.__version__` -* to choose a name and prefix for a 'setup.py sdist' tarball - -## Theory of Operation - -Versioneer works by adding a special `_version.py` file into your source -tree, where your `__init__.py` can import it. This `_version.py` knows how to -dynamically ask the VCS tool for version information at import time. - -`_version.py` also contains `$Revision$` markers, and the installation -process marks `_version.py` to have this marker rewritten with a tag name -during the `git archive` command. As a result, generated tarballs will -contain enough information to get the proper version. - -To allow `setup.py` to compute a version too, a `versioneer.py` is added to -the top level of your source tree, next to `setup.py` and the `setup.cfg` -that configures it. This overrides several distutils/setuptools commands to -compute the version when invoked, and changes `setup.py build` and `setup.py -sdist` to replace `_version.py` with a small static file that contains just -the generated version data. - -## Installation - -See [INSTALL.md](./INSTALL.md) for detailed installation instructions. - -## Version-String Flavors - -Code which uses Versioneer can learn about its version string at runtime by -importing `_version` from your main `__init__.py` file and running the -`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can -import the top-level `versioneer.py` and run `get_versions()`. - -Both functions return a dictionary with different flavors of version -information: - -* `['version']`: A condensed version string, rendered using the selected - style. This is the most commonly used value for the project's version - string. The default "pep440" style yields strings like `0.11`, - `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section - below for alternative styles. - -* `['full-revisionid']`: detailed revision identifier. For Git, this is the - full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". - -* `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the - commit date in ISO 8601 format. This will be None if the date is not - available. - -* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that - this is only accurate if run in a VCS checkout, otherwise it is likely to - be False or None - -* `['error']`: if the version string could not be computed, this will be set - to a string describing the problem, otherwise it will be None. It may be - useful to throw an exception in setup.py if this is set, to avoid e.g. - creating tarballs with a version string of "unknown". - -Some variants are more useful than others. Including `full-revisionid` in a -bug report should allow developers to reconstruct the exact code being tested -(or indicate the presence of local changes that should be shared with the -developers). `version` is suitable for display in an "about" box or a CLI -`--version` output: it can be easily compared against release notes and lists -of bugs fixed in various releases. - -The installer adds the following text to your `__init__.py` to place a basic -version in `YOURPROJECT.__version__`: - - from ._version import get_versions - __version__ = get_versions()['version'] - del get_versions - -## Styles - -The setup.cfg `style=` configuration controls how the VCS information is -rendered into a version string. - -The default style, "pep440", produces a PEP440-compliant string, equal to the -un-prefixed tag name for actual releases, and containing an additional "local -version" section with more detail for in-between builds. For Git, this is -TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags ---dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the -tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and -that this commit is two revisions ("+2") beyond the "0.11" tag. For released -software (exactly equal to a known tag), the identifier will only contain the -stripped tag, e.g. "0.11". - -Other styles are available. See [details.md](details.md) in the Versioneer -source tree for descriptions. - -## Debugging - -Versioneer tries to avoid fatal errors: if something goes wrong, it will tend -to return a version of "0+unknown". To investigate the problem, run `setup.py -version`, which will run the version-lookup code in a verbose mode, and will -display the full contents of `get_versions()` (including the `error` string, -which may help identify what went wrong). - -## Known Limitations - -Some situations are known to cause problems for Versioneer. This details the -most significant ones. More can be found on Github -[issues page](https://github.com/python-versioneer/python-versioneer/issues). - -### Subprojects - -Versioneer has limited support for source trees in which `setup.py` is not in -the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are -two common reasons why `setup.py` might not be in the root: - -* Source trees which contain multiple subprojects, such as - [Buildbot](https://github.com/buildbot/buildbot), which contains both - "master" and "slave" subprojects, each with their own `setup.py`, - `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI - distributions (and upload multiple independently-installable tarballs). -* Source trees whose main purpose is to contain a C library, but which also - provide bindings to Python (and perhaps other languages) in subdirectories. - -Versioneer will look for `.git` in parent directories, and most operations -should get the right version string. However `pip` and `setuptools` have bugs -and implementation details which frequently cause `pip install .` from a -subproject directory to fail to find a correct version string (so it usually -defaults to `0+unknown`). - -`pip install --editable .` should work correctly. `setup.py install` might -work too. - -Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in -some later version. - -[Bug #38](https://github.com/python-versioneer/python-versioneer/issues/38) is tracking -this issue. The discussion in -[PR #61](https://github.com/python-versioneer/python-versioneer/pull/61) describes the -issue from the Versioneer side in more detail. -[pip PR#3176](https://github.com/pypa/pip/pull/3176) and -[pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve -pip to let Versioneer work correctly. - -Versioneer-0.16 and earlier only looked for a `.git` directory next to the -`setup.cfg`, so subprojects were completely unsupported with those releases. - -### Editable installs with setuptools <= 18.5 - -`setup.py develop` and `pip install --editable .` allow you to install a -project into a virtualenv once, then continue editing the source code (and -test) without re-installing after every change. - -"Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a -convenient way to specify executable scripts that should be installed along -with the python package. - -These both work as expected when using modern setuptools. When using -setuptools-18.5 or earlier, however, certain operations will cause -`pkg_resources.DistributionNotFound` errors when running the entrypoint -script, which must be resolved by re-installing the package. This happens -when the install happens with one version, then the egg_info data is -regenerated while a different version is checked out. Many setup.py commands -cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into -a different virtualenv), so this can be surprising. - -[Bug #83](https://github.com/python-versioneer/python-versioneer/issues/83) describes -this one, but upgrading to a newer version of setuptools should probably -resolve it. - - -## Updating Versioneer - -To upgrade your project to a new release of Versioneer, do the following: - -* install the new Versioneer (`pip install -U versioneer` or equivalent) -* edit `setup.cfg`, if necessary, to include any new configuration settings - indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. -* re-run `versioneer install` in your source tree, to replace - `SRC/_version.py` -* commit any changed files - -## Future Directions - -This tool is designed to make it easily extended to other version-control -systems: all VCS-specific components are in separate directories like -src/git/ . The top-level `versioneer.py` script is assembled from these -components by running make-versioneer.py . In the future, make-versioneer.py -will take a VCS name as an argument, and will construct a version of -`versioneer.py` that is specific to the given VCS. It might also take the -configuration arguments that are currently provided manually during -installation by editing setup.py . Alternatively, it might go the other -direction and include code from all supported VCS systems, reducing the -number of intermediate scripts. - -## Similar projects - -* [setuptools_scm](https://github.com/pypa/setuptools_scm/) - a non-vendored build-time - dependency -* [minver](https://github.com/jbweston/miniver) - a lightweight reimplementation of - versioneer -* [versioningit](https://github.com/jwodder/versioningit) - a PEP 518-based setuptools - plugin - -## License - -To make Versioneer easier to embed, all its code is dedicated to the public -domain. The `_version.py` that it creates is also in the public domain. -Specifically, both are released under the Creative Commons "Public Domain -Dedication" license (CC0-1.0), as described in -https://creativecommons.org/publicdomain/zero/1.0/ . - -[pypi-image]: https://img.shields.io/pypi/v/versioneer.svg -[pypi-url]: https://pypi.python.org/pypi/versioneer/ -[travis-image]: -https://img.shields.io/travis/com/python-versioneer/python-versioneer.svg -[travis-url]: https://travis-ci.com/github/python-versioneer/python-versioneer - -""" - -import configparser -import errno -import json -import os -import re -import subprocess -import sys - - -class VersioneerConfig: # pylint: disable=too-few-public-methods # noqa - """Container for Versioneer configuration parameters.""" - - -def get_root(): - """Get the project root directory. - - We require that all commands are run from the project root, i.e. the - directory that contains setup.py, setup.cfg, and versioneer.py . - """ - root = os.path.realpath(os.path.abspath(os.getcwd())) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - # allow 'python path/to/setup.py COMMAND' - root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - err = ("Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND').") - raise VersioneerBadRootError(err) - try: - # Certain runtime workflows (setup.py install/develop in a setuptools - # tree) execute all dependencies in a single python process, so - # "versioneer" may be imported multiple times, and python's shared - # module-import table will cache the first one. So we can't use - # os.path.dirname(__file__), as that will find whichever - # versioneer.py was first imported, even in later projects. - my_path = os.path.realpath(os.path.abspath(__file__)) - me_dir = os.path.normcase(os.path.splitext(my_path)[0]) - vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) - if me_dir != vsr_dir: - print("Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(my_path), versioneer_py)) - except NameError: - pass - return root - - -def get_config_from_root(root): - """Read the project setup.cfg file to determine Versioneer config.""" - # This might raise EnvironmentError (if setup.cfg is missing), or - # configparser.NoSectionError (if it lacks a [versioneer] section), or - # configparser.NoOptionError (if it lacks "VCS="). See the docstring at - # the top of versioneer.py for instructions on writing your setup.cfg . - setup_cfg = os.path.join(root, "setup.cfg") - parser = configparser.ConfigParser() - with open(setup_cfg, "r") as cfg_file: - parser.read_file(cfg_file) - VCS = parser.get("versioneer", "VCS") # mandatory - - # Dict-like interface for non-mandatory entries - section = parser["versioneer"] - - # pylint:disable=attribute-defined-outside-init # noqa - cfg = VersioneerConfig() - cfg.VCS = VCS - cfg.style = section.get("style", "") - cfg.versionfile_source = section.get("versionfile_source") - cfg.versionfile_build = section.get("versionfile_build") - cfg.tag_prefix = section.get("tag_prefix") - if cfg.tag_prefix in ("''", '""'): - cfg.tag_prefix = "" - cfg.parentdir_prefix = section.get("parentdir_prefix") - cfg.verbose = section.get("verbose") - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -# these dictionaries contain VCS-specific tools -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Create decorator to mark a method as the handler of a VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - HANDLERS.setdefault(vcs, {})[method] = f - return f - - return decorate - - -# pylint:disable=too-many-arguments,consider-using-with # noqa -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - process = None - for command in commands: - try: - dispcmd = str([command] + args) - # remember shell=False, so use git.cmd on windows, not just git - process = subprocess.Popen([command] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None, None - stdout = process.communicate()[0].strip().decode() - if process.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, process.returncode - return stdout, process.returncode - - -LONG_VERSION_PY['git'] = r''' -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.20 (https://github.com/python-versioneer/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" - git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" - git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: # pylint: disable=too-few-public-methods - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "%(STYLE)s" - cfg.tag_prefix = "%(TAG_PREFIX)s" - cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" - cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Create decorator to mark a method as the handler of a VCS.""" - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - return decorate - - -# pylint:disable=too-many-arguments,consider-using-with # noqa -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - process = None - for command in commands: - try: - dispcmd = str([command] + args) - # remember shell=False, so use git.cmd on windows, not just git - process = subprocess.Popen([command] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %%s" %% dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %%s" %% (commands,)) - return None, None - stdout = process.communicate()[0].strip().decode() - if process.returncode != 0: - if verbose: - print("unable to run %%s (error)" %% dispcmd) - print("stdout was %%s" %% stdout) - return None, process.returncode - return stdout, process.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for _ in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print("Tried directories %%s but none started with prefix %%s" %% - (str(rootdirs), parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - with open(versionfile_abs, "r") as fobj: - for line in fobj: - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if "refnames" not in keywords: - raise NotThisMethod("Short version file found") - date = keywords.get("date") - if date is not None: - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - - # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = {r.strip() for r in refnames.strip("()").split(",")} - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %%d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r'\d', r)} - if verbose: - print("discarding '%%s', no digits" %% ",".join(refs - tags)) - if verbose: - print("likely tags: %%s" %% ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - # Filter out refs that exactly match prefix or that don't start - # with a number once the prefix is stripped (mostly a concern - # when prefix is '') - if not re.match(r'\d', r): - continue - if verbose: - print("picking %%s" %% r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) - if rc != 0: - if verbose: - print("Directory %%s not under git control" %% root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = runner(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%%s*" %% tag_prefix], - cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], - cwd=root) - # --abbrev-ref was added in git-1.6.3 - if rc != 0 or branch_name is None: - raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") - branch_name = branch_name.strip() - - if branch_name == "HEAD": - # If we aren't exactly on a branch, pick a branch which represents - # the current commit. If all else fails, we are on a branchless - # commit. - branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) - # --contains was added in git-1.5.4 - if rc != 0 or branches is None: - raise NotThisMethod("'git branch --contains' returned error") - branches = branches.split("\n") - - # Remove the first line if we're running detached - if "(" in branches[0]: - branches.pop(0) - - # Strip off the leading "* " from the list of branches. - branches = [branch[2:] for branch in branches] - if "master" in branches: - branch_name = "master" - elif not branches: - branch_name = None - else: - # Pick the first branch that is returned. Good or bad. - branch_name = branches[0] - - pieces["branch"] = branch_name - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%%s'" - %% describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%%s' doesn't start with prefix '%%s'" - print(fmt %% (full_tag, tag_prefix)) - pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" - %% (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = runner(GITS, ["rev-list", "HEAD", "--count"], cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = runner(GITS, ["show", "-s", "--format=%%ci", "HEAD"], cwd=root)[0].strip() - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_branch(pieces): - """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . - - The ".dev0" means not master branch. Note that .dev0 sorts backwards - (a feature branch will appear "older" than the master branch). - - Exceptions: - 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0" - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+untagged.%%d.g%%s" %% (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post0.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post0.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post0.dev%%d" %% pieces["distance"] - else: - # exception #1 - rendered = "0.post0.dev%%d" %% pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%%s" %% pieces["short"] - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%%s" %% pieces["short"] - return rendered - - -def render_pep440_post_branch(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . - - The ".dev0" means not master branch. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%%s" %% pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+g%%s" %% pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-branch": - rendered = render_pep440_branch(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-post-branch": - rendered = render_pep440_post_branch(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%%s'" %% style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for _ in cfg.versionfile_source.split('/'): - root = os.path.dirname(root) - except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None} - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", "date": None} -''' - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - with open(versionfile_abs, "r") as fobj: - for line in fobj: - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if "refnames" not in keywords: - raise NotThisMethod("Short version file found") - date = keywords.get("date") - if date is not None: - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = {r.strip() for r in refnames.strip("()").split(",")} - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r'\d', r)} - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - # Filter out refs that exactly match prefix or that don't start - # with a number once the prefix is stripped (mostly a concern - # when prefix is '') - if not re.match(r'\d', r): - continue - if verbose: - print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = runner(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], - cwd=root) - # --abbrev-ref was added in git-1.6.3 - if rc != 0 or branch_name is None: - raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") - branch_name = branch_name.strip() - - if branch_name == "HEAD": - # If we aren't exactly on a branch, pick a branch which represents - # the current commit. If all else fails, we are on a branchless - # commit. - branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) - # --contains was added in git-1.5.4 - if rc != 0 or branches is None: - raise NotThisMethod("'git branch --contains' returned error") - branches = branches.split("\n") - - # Remove the first line if we're running detached - if "(" in branches[0]: - branches.pop(0) - - # Strip off the leading "* " from the list of branches. - branches = [branch[2:] for branch in branches] - if "master" in branches: - branch_name = "master" - elif not branches: - branch_name = None - else: - # Pick the first branch that is returned. Good or bad. - branch_name = branches[0] - - pieces["branch"] = branch_name - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = runner(GITS, ["rev-list", "HEAD", "--count"], cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def do_vcs_install(manifest_in, versionfile_source, ipy): - """Git-specific installation logic for Versioneer. - - For Git, this means creating/changing .gitattributes to mark _version.py - for export-subst keyword substitution. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - files = [manifest_in, versionfile_source] - if ipy: - files.append(ipy) - try: - my_path = __file__ - if my_path.endswith(".pyc") or my_path.endswith(".pyo"): - my_path = os.path.splitext(my_path)[0] + ".py" - versioneer_file = os.path.relpath(my_path) - except NameError: - versioneer_file = "versioneer.py" - files.append(versioneer_file) - present = False - try: - with open(".gitattributes", "r") as fobj: - for line in fobj: - if line.strip().startswith(versionfile_source): - if "export-subst" in line.strip().split()[1:]: - present = True - break - except EnvironmentError: - pass - if not present: - with open(".gitattributes", "a+") as fobj: - fobj.write("{versionfile_source} export-subst\n".format(versionfile_source=versionfile_source)) - files.append(".gitattributes") - run_command(GITS, ["add", "--"] + files) - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for _ in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print("Tried directories %s but none started with prefix %s" % - (str(rootdirs), parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.20) from -# revision-control system data, or from the parent directory name of an -# unpacked source archive. Distribution tarballs contain a pre-generated copy -# of this file. - -import json - -version_json = ''' -%s -''' # END VERSION_JSON - - -def get_versions(): - return json.loads(version_json) -""" - - -def versions_from_file(filename): - """Try to determine the version from _version.py if present.""" - try: - with open(filename) as f: - contents = f.read() - except EnvironmentError: - raise NotThisMethod("unable to read _version.py") - mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) - if not mo: - mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) - if not mo: - raise NotThisMethod("no version_json in _version.py") - return json.loads(mo.group(1)) - - -def write_to_version_file(filename, versions): - """Write the given version number to the given _version.py file.""" - if os.path.isfile(filename): - os.unlink(filename) - contents = json.dumps(versions, sort_keys=True, - indent=1, separators=(",", ": ")) - with open(filename, "w") as f: - f.write(SHORT_VERSION_PY % contents) - - print("set %s to '%s'" % (filename, versions["version"])) - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_branch(pieces): - """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . - - The ".dev0" means not master branch. Note that .dev0 sorts backwards - (a feature branch will appear "older" than the master branch). - - Exceptions: - 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0" - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post0.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post0.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post0.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post0.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_post_branch(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . - - The ".dev0" means not master branch. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-branch": - rendered = render_pep440_branch(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-post-branch": - rendered = render_pep440_post_branch(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} - - -class VersioneerBadRootError(Exception): - """The project root directory is unknown or missing key files.""" - - -def get_versions(verbose=False): - """Get the project version from whatever source is available. - - Returns dict with two keys: 'version' and 'full'. - """ - if "versioneer" in sys.modules: - # see the discussion in cmdclass.py:get_cmdclass() - del sys.modules["versioneer"] - - root = get_root() - cfg = get_config_from_root(root) - - assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" - handlers = HANDLERS.get(cfg.VCS) - assert handlers, "unrecognized VCS '%s'" % cfg.VCS - verbose = verbose or cfg.verbose - assert cfg.versionfile_source is not None, \ - "please set versioneer.versionfile_source" - assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" - - versionfile_abs = os.path.join(root, cfg.versionfile_source) - - # extract version from first of: _version.py, VCS command (e.g. 'git - # describe'), parentdir. This is meant to work for developers using a - # source checkout, for users of a tarball created by 'setup.py sdist', - # and for users of a tarball/zipball created by 'git archive' or github's - # download-from-tag feature or the equivalent in other VCSes. - - get_keywords_f = handlers.get("get_keywords") - from_keywords_f = handlers.get("keywords") - if get_keywords_f and from_keywords_f: - try: - keywords = get_keywords_f(versionfile_abs) - ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) - if verbose: - print("got version from expanded keyword %s" % ver) - return ver - except NotThisMethod: - pass - - try: - ver = versions_from_file(versionfile_abs) - if verbose: - print("got version from file %s %s" % (versionfile_abs, ver)) - return ver - except NotThisMethod: - pass - - from_vcs_f = handlers.get("pieces_from_vcs") - if from_vcs_f: - try: - pieces = from_vcs_f(cfg.tag_prefix, root, verbose) - ver = render(pieces, cfg.style) - if verbose: - print("got version from VCS %s" % ver) - return ver - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - if verbose: - print("got version from parentdir %s" % ver) - return ver - except NotThisMethod: - pass - - if os.path.isfile(".version"): - try: - version = open(".version").readlines()[0].strip() - return {"version": version, "full-revisionid": None, - "dirty": None, "error": None, - "date": None} - except Exception as e: - pass - - if verbose: - print("unable to compute version") - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, "error": "unable to compute version", - "date": None} - - -def get_version(): - """Get the short version string for this project.""" - return get_versions()["version"] - - -def get_cmdclass(cmdclass=None): - """Get the custom setuptools/distutils subclasses used by Versioneer. - - If the package uses a different cmdclass (e.g. one from numpy), it - should be provide as an argument. - """ - if "versioneer" in sys.modules: - del sys.modules["versioneer"] - # this fixes the "python setup.py develop" case (also 'install' and - # 'easy_install .'), in which subdependencies of the main project are - # built (using setup.py bdist_egg) in the same python process. Assume - # a main project A and a dependency B, which use different versions - # of Versioneer. A's setup.py imports A's Versioneer, leaving it in - # sys.modules by the time B's setup.py is executed, causing B to run - # with the wrong versioneer. Setuptools wraps the sub-dep builds in a - # sandbox that restores sys.modules to it's pre-build state, so the - # parent is protected against the child's "import versioneer". By - # removing ourselves from sys.modules here, before the child build - # happens, we protect the child from the parent's versioneer too. - # Also see https://github.com/python-versioneer/python-versioneer/issues/52 - - cmds = {} if cmdclass is None else cmdclass.copy() - - # we add "version" to both distutils and setuptools - from distutils.core import Command - - class cmd_version(Command): - description = "report generated version string" - user_options = [] - boolean_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - vers = get_versions(verbose=True) - print("Version: %s" % vers["version"]) - print(" full-revisionid: %s" % vers.get("full-revisionid")) - print(" dirty: %s" % vers.get("dirty")) - print(" date: %s" % vers.get("date")) - if vers["error"]: - print(" error: %s" % vers["error"]) - - cmds["version"] = cmd_version - - # we override "build_py" in both distutils and setuptools - # - # most invocation pathways end up running build_py: - # distutils/build -> build_py - # distutils/install -> distutils/build ->.. - # setuptools/bdist_wheel -> distutils/install ->.. - # setuptools/bdist_egg -> distutils/install_lib -> build_py - # setuptools/install -> bdist_egg ->.. - # setuptools/develop -> ? - # pip install: - # copies source tree to a tempdir before running egg_info/etc - # if .git isn't copied too, 'git describe' will fail - # then does setup.py bdist_wheel, or sometimes setup.py install - # setup.py egg_info -> ? - - # we override different "build_py" commands for both environments - if 'build_py' in cmds: - _build_py = cmds['build_py'] - elif "setuptools" in sys.modules: - from setuptools.command.build_py import build_py as _build_py - else: - from distutils.command.build_py import build_py as _build_py - - class cmd_build_py(_build_py): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - _build_py.run(self) - # now locate _version.py in the new build/ directory and replace - # it with an updated value - if cfg.versionfile_build: - target_versionfile = os.path.join(self.build_lib, - cfg.versionfile_build) - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - cmds["build_py"] = cmd_build_py - - if 'build_ext' in cmds: - _build_ext = cmds['build_ext'] - elif "setuptools" in sys.modules: - from setuptools.command.build_ext import build_ext as _build_ext - else: - from distutils.command.build_ext import build_ext as _build_ext - - class cmd_build_ext(_build_ext): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - _build_ext.run(self) - if self.inplace: - # build_ext --inplace will only build extensions in - # build/lib<..> dir with no _version.py to write to. - # As in place builds will already have a _version.py - # in the module dir, we do not need to write one. - return - # now locate _version.py in the new build/ directory and replace - # it with an updated value - target_versionfile = os.path.join(self.build_lib, - cfg.versionfile_build) - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - cmds["build_ext"] = cmd_build_ext - - if "cx_Freeze" in sys.modules: # cx_freeze enabled? - from cx_Freeze.dist import build_exe as _build_exe - # nczeczulin reports that py2exe won't like the pep440-style string - # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. - # setup(console=[{ - # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION - # "product_version": versioneer.get_version(), - # ... - - class cmd_build_exe(_build_exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _build_exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) - - cmds["build_exe"] = cmd_build_exe - del cmds["build_py"] - - if 'py2exe' in sys.modules: # py2exe enabled? - from py2exe.distutils_buildexe import py2exe as _py2exe - - class cmd_py2exe(_py2exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _py2exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) - - cmds["py2exe"] = cmd_py2exe - - # we override different "sdist" commands for both environments - if 'sdist' in cmds: - _sdist = cmds['sdist'] - elif "setuptools" in sys.modules: - from setuptools.command.sdist import sdist as _sdist - else: - from distutils.command.sdist import sdist as _sdist - - class cmd_sdist(_sdist): - def run(self): - versions = get_versions() - # pylint:disable=attribute-defined-outside-init # noqa - self._versioneer_generated_versions = versions - # unless we update this, the command will keep using the old - # version - self.distribution.metadata.version = versions["version"] - return _sdist.run(self) - - def make_release_tree(self, base_dir, files): - root = get_root() - cfg = get_config_from_root(root) - _sdist.make_release_tree(self, base_dir, files) - # now locate _version.py in the new base_dir directory - # (remembering that it may be a hardlink) and replace it with an - # updated value - target_versionfile = os.path.join(base_dir, cfg.versionfile_source) - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, - self._versioneer_generated_versions) - - cmds["sdist"] = cmd_sdist - - return cmds - - -CONFIG_ERROR = """ -setup.cfg is missing the necessary Versioneer configuration. You need -a section like: - - [versioneer] - VCS = git - style = pep440 - versionfile_source = src/myproject/_version.py - versionfile_build = myproject/_version.py - tag_prefix = - parentdir_prefix = myproject- - -You will also need to edit your setup.py to use the results: - - import versioneer - setup(version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), ...) - -Please read the docstring in ./versioneer.py for configuration instructions, -edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. -""" - -SAMPLE_CONFIG = """ -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. - -[versioneer] -#VCS = git -#style = pep440 -#versionfile_source = -#versionfile_build = -#tag_prefix = -#parentdir_prefix = - -""" - -OLD_SNIPPET = """ -from ._version import get_versions -__version__ = get_versions()['version'] -del get_versions -""" - -INIT_PY_SNIPPET = """ -from . import {0} -__version__ = {0}.get_versions()['version'] -""" - - -def do_setup(): - """Do main VCS-independent setup function for installing Versioneer.""" - root = get_root() - try: - cfg = get_config_from_root(root) - except (EnvironmentError, configparser.NoSectionError, - configparser.NoOptionError) as e: - if isinstance(e, (EnvironmentError, configparser.NoSectionError)): - print("Adding sample versioneer config to setup.cfg", - file=sys.stderr) - with open(os.path.join(root, "setup.cfg"), "a") as f: - f.write(SAMPLE_CONFIG) - print(CONFIG_ERROR, file=sys.stderr) - return 1 - - print(" creating %s" % cfg.versionfile_source) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), - "__init__.py") - if os.path.exists(ipy): - try: - with open(ipy, "r") as f: - old = f.read() - except EnvironmentError: - old = "" - module = os.path.splitext(os.path.basename(cfg.versionfile_source))[0] - snippet = INIT_PY_SNIPPET.format(module) - if OLD_SNIPPET in old: - print(" replacing boilerplate in %s" % ipy) - with open(ipy, "w") as f: - f.write(old.replace(OLD_SNIPPET, snippet)) - elif snippet not in old: - print(" appending to %s" % ipy) - with open(ipy, "a") as f: - f.write(snippet) - else: - print(" %s unmodified" % ipy) - else: - print(" %s doesn't exist, ok" % ipy) - ipy = None - - # Make sure both the top-level "versioneer.py" and versionfile_source - # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so - # they'll be copied into source distributions. Pip won't be able to - # install the package without this. - manifest_in = os.path.join(root, "MANIFEST.in") - simple_includes = set() - try: - with open(manifest_in, "r") as f: - for line in f: - if line.startswith("include "): - for include in line.split()[1:]: - simple_includes.add(include) - except EnvironmentError: - pass - # That doesn't cover everything MANIFEST.in can do - # (http://docs.python.org/2/distutils/sourcedist.html#commands), so - # it might give some false negatives. Appending redundant 'include' - # lines is safe, though. - if "versioneer.py" not in simple_includes: - print(" appending 'versioneer.py' to MANIFEST.in") - with open(manifest_in, "a") as f: - f.write("include versioneer.py\n") - else: - print(" 'versioneer.py' already in MANIFEST.in") - if cfg.versionfile_source not in simple_includes: - print(" appending versionfile_source ('%s') to MANIFEST.in" % - cfg.versionfile_source) - with open(manifest_in, "a") as f: - f.write("include %s\n" % cfg.versionfile_source) - else: - print(" versionfile_source already in MANIFEST.in") - - # Make VCS-specific changes. For git, this means creating/changing - # .gitattributes to mark _version.py for export-subst keyword - # substitution. - do_vcs_install(manifest_in, cfg.versionfile_source, ipy) - return 0 - - -def scan_setup_py(): - """Validate the contents of setup.py against Versioneer's expectations.""" - found = set() - setters = False - errors = 0 - with open("setup.py", "r") as f: - for line in f.readlines(): - if "import versioneer" in line: - found.add("import") - if "versioneer.get_cmdclass()" in line: - found.add("cmdclass") - if "versioneer.get_version()" in line: - found.add("get_version") - if "versioneer.VCS" in line: - setters = True - if "versioneer.versionfile_source" in line: - setters = True - if len(found) != 3: - print("") - print("Your setup.py appears to be missing some important items") - print("(but I might be wrong). Please make sure it has something") - print("roughly like the following:") - print("") - print(" import versioneer") - print(" setup( version=versioneer.get_version(),") - print(" cmdclass=versioneer.get_cmdclass(), ...)") - print("") - errors += 1 - if setters: - print("You should remove lines like 'versioneer.VCS = ' and") - print("'versioneer.versionfile_source = ' . This configuration") - print("now lives in setup.cfg, and should be removed from setup.py") - print("") - errors += 1 - return errors - - -if __name__ == "__main__": - cmd = sys.argv[1] - if cmd == "setup": - errors = do_setup() - errors += scan_setup_py() - if errors: - sys.exit(1) From 17aba14ffb8f055b09e46aaa9669476fe7ff8f5f Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Fri, 4 Sep 2026 13:56:31 +0200 Subject: [PATCH 13/30] build: expose CLI as console entry points, cap python at <3.14 bin/ scripts -> src/pyace/cli/*.py with [project.scripts] entry points. This retires setuptools' deprecated script-files, and makes the commands work on Windows, where script-files are installed without an executable wrapper. The scripts had three different shapes: - main() already: pace_activeset, pace_augment, pace_corerep, pace_select (valid entry points unchanged) - main(args): pacemaker, pace_collect -> main(args=None), since argparse already received `args` and falls back to sys.argv[1:] when it is None - no main at all, just module-level script body: pace_info, pace_timing, pace_yaml2yace -> body wrapped in main() Shebangs dropped (they are modules now); the __main__ guards are kept, so `python -m pyace.cli.pacemaker` still works alongside the installed command. Nothing outside pyproject.toml referenced the bin/ paths -- the CLI integration tests already invoked the commands by name, so they are unaffected. requires-python gains an upper bound of <3.14. grace-tensorpotential, which pyace must stay installable beside, declares only ">=3.9", so the real ceiling comes from its tensorflow<=2.20 dependency: tensorflow 2.20 ships wheels for 3.9-3.13 and nothing newer. This needs revisiting when tensorflow adds 3.14. Verified: 9 entry points registered, every command's --help runs, pacemaker -v reports 0.4.0, `python -m pyace.cli.pace_info` still works, tests/test-CLI/Cu-I/integration_test.sh passes end to end, and 292 tests pass. --- bin/pace_timing | 23 ------------ bin/pace_yaml2yace | 32 ---------------- pyproject.toml | 33 ++++++++--------- src/pyace/cli/__init__.py | 6 +++ .../pyace/cli/pace_activeset.py | 1 - .../pyace/cli/pace_augment.py | 1 - .../pyace/cli/pace_collect.py | 5 +-- .../pyace/cli/pace_corerep.py | 1 - bin/pace_info => src/pyace/cli/pace_info.py | 22 ++++++----- .../pyace/cli/pace_select.py | 1 - src/pyace/cli/pace_timing.py | 28 ++++++++++++++ src/pyace/cli/pace_yaml2yace.py | 37 +++++++++++++++++++ bin/pacemaker => src/pyace/cli/pacemaker.py | 5 +-- 13 files changed, 104 insertions(+), 91 deletions(-) delete mode 100644 bin/pace_timing delete mode 100644 bin/pace_yaml2yace create mode 100644 src/pyace/cli/__init__.py rename bin/pace_activeset => src/pyace/cli/pace_activeset.py (99%) rename bin/pace_augment => src/pyace/cli/pace_augment.py (99%) rename bin/pace_collect => src/pyace/cli/pace_collect.py (99%) rename bin/pace_corerep => src/pyace/cli/pace_corerep.py (99%) rename bin/pace_info => src/pyace/cli/pace_info.py (88%) rename bin/pace_select => src/pyace/cli/pace_select.py (99%) create mode 100644 src/pyace/cli/pace_timing.py create mode 100644 src/pyace/cli/pace_yaml2yace.py rename bin/pacemaker => src/pyace/cli/pacemaker.py (99%) diff --git a/bin/pace_timing b/bin/pace_timing deleted file mode 100644 index 4fb283f..0000000 --- a/bin/pace_timing +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python -import argparse -import numpy as np - -from pyace.utils.timing import run_timing_test - -parser = argparse.ArgumentParser(prog="pace_timing", - description="Utility to run the single-CPU timing test for PACE (.yaml) potential ") - -parser.add_argument("potential_file", help="B-basis file name (.yaml)", type=str, nargs='+', default=[]) -parser.add_argument("-n", "--nstructures", help="number of structures", type=int, default=10) - -args_parse = parser.parse_args() -potential_files = args_parse.potential_file -nstructures = args_parse.nstructures - -for potential_file in potential_files: - np.random.seed(42) - print() - print("**************************************************") - print("Using potential file: ", potential_file) - print("**************************************************") - run_timing_test(potential_file, n_struct=nstructures) diff --git a/bin/pace_yaml2yace b/bin/pace_yaml2yace deleted file mode 100644 index a73c7bd..0000000 --- a/bin/pace_yaml2yace +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python -import argparse - -from pyace import ACEBBasisSet - -parser = argparse.ArgumentParser(prog="pace_yaml2yace", - description="Conversion utility from B-basis (.yaml file) to new-style Ctilde-basis (.yace file)") -parser.add_argument("input", help="input B-basis file name (.yaml)", type=str, nargs='+', default=[]) -parser.add_argument("-o", "--output", help="output Ctilde-basis file name (.yace)", type=str, default="") - -args_parse = parser.parse_args() -input_yaml_filenames = args_parse.input -output_ace_filename = args_parse.output - - -for input_yaml_filename in input_yaml_filenames: - if output_ace_filename == "": - if input_yaml_filename.endswith("yaml"): - actual_output_ace_filename = input_yaml_filename.replace("yaml", "yace") - elif input_yaml_filename.endswith("yml"): - actual_output_ace_filename = input_yaml_filename.replace("yml", "yace") - else: - actual_output_ace_filename = input_yaml_filename + ".yace" - else: - actual_output_ace_filename = output_ace_filename - - print("Loading B-basis from '{}'".format(input_yaml_filename)) - bbasis = ACEBBasisSet(input_yaml_filename) - print("Converting to Ctilde-basis") - cbasis = bbasis.to_ACECTildeBasisSet() - print("Saving Ctilde-basis to '{}'".format(actual_output_ace_filename)) - cbasis.save_yaml(actual_output_ace_filename) diff --git a/pyproject.toml b/pyproject.toml index 1276e59..9896b2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,11 @@ authors = [ ] description = 'Python bindings, utilities for PACE and fitting code "pacemaker"' readme = "README.md" -requires-python = ">=3.9" +# Upper bound tracks grace-tensorpotential, which pyace must stay installable +# alongside. It declares only ">=3.9", so the real ceiling is its tensorflow<=2.20 +# dependency: tensorflow 2.20 publishes wheels for 3.9-3.13 and nothing newer, +# so a grace environment cannot exist on 3.14. Revisit when tensorflow adds it. +requires-python = ">=3.9,<3.14" license = { text = "Academic Software License (ASL)" } classifiers = [ "License :: Other/Proprietary License", @@ -53,24 +57,19 @@ dependencies = [ [project.urls] "Homepage" = "https://github.com/ICAMS/python-ace" +[project.scripts] +pacemaker = "pyace.cli.pacemaker:main" +pace_activeset = "pyace.cli.pace_activeset:main" +pace_augment = "pyace.cli.pace_augment:main" +pace_collect = "pyace.cli.pace_collect:main" +pace_corerep = "pyace.cli.pace_corerep:main" +pace_info = "pyace.cli.pace_info:main" +pace_select = "pyace.cli.pace_select:main" +pace_timing = "pyace.cli.pace_timing:main" +pace_yaml2yace = "pyace.cli.pace_yaml2yace:main" + [tool.setuptools] zip-safe = false -# These are plain scripts with a shebang, not importable main() entry points, -# so they are installed as files. Converting them to [project.scripts] console -# entry points would also make them work on Windows, but needs each bin/ script -# refactored into a module first. -script-files = [ - "bin/pacemaker", - "bin/pace_yaml2yace", - "bin/pace_timing", - "bin/pace_info", - "bin/pace_activeset", - "bin/pace_select", - "bin/pace_collect", - "bin/pace_augment", - "bin/pace_corerep", -] - [tool.setuptools.packages.find] where = ["src"] diff --git a/src/pyace/cli/__init__.py b/src/pyace/cli/__init__.py new file mode 100644 index 0000000..f236c6c --- /dev/null +++ b/src/pyace/cli/__init__.py @@ -0,0 +1,6 @@ +"""Command-line entry points for pyace. + +These were standalone files under bin/. They are modules now so that each one +can be exposed as a console entry point in pyproject.toml, which also makes +them work on Windows, where setuptools' script-files are not executable. +""" diff --git a/bin/pace_activeset b/src/pyace/cli/pace_activeset.py similarity index 99% rename from bin/pace_activeset rename to src/pyace/cli/pace_activeset.py index 8797bbc..cb363e5 100644 --- a/bin/pace_activeset +++ b/src/pyace/cli/pace_activeset.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python import argparse import logging import sys diff --git a/bin/pace_augment b/src/pyace/cli/pace_augment.py similarity index 99% rename from bin/pace_augment rename to src/pyace/cli/pace_augment.py index 05b2674..7f73d20 100644 --- a/bin/pace_augment +++ b/src/pyace/cli/pace_augment.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python import logging import sys diff --git a/bin/pace_collect b/src/pyace/cli/pace_collect.py similarity index 99% rename from bin/pace_collect rename to src/pyace/cli/pace_collect.py index d13097c..0dd4712 100644 --- a/bin/pace_collect +++ b/src/pyace/cli/pace_collect.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python import glob import sys @@ -171,7 +170,7 @@ def get_free_atom_energy(df, el): return ref_epa -def main(args): +def main(args=None): ############################################################################################## parser = argparse.ArgumentParser() @@ -307,4 +306,4 @@ def collect_and_update(root, fname, format): if __name__ == "__main__": - main(sys.argv[1:]) + main() diff --git a/bin/pace_corerep b/src/pyace/cli/pace_corerep.py similarity index 99% rename from bin/pace_corerep rename to src/pyace/cli/pace_corerep.py index 195487a..ca093b0 100644 --- a/bin/pace_corerep +++ b/src/pyace/cli/pace_corerep.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python import os.path import sys import argparse diff --git a/bin/pace_info b/src/pyace/cli/pace_info.py similarity index 88% rename from bin/pace_info rename to src/pyace/cli/pace_info.py index fa92211..03f918b 100644 --- a/bin/pace_info +++ b/src/pyace/cli/pace_info.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python import argparse from collections import defaultdict @@ -119,14 +118,19 @@ def analyse_potential_shape(ace_potential): print("=" * 40) -parser = argparse.ArgumentParser(prog="pace_info", - description="Utility to analyze PACE (.yaml) potential shape and other parameters") +def main(): + parser = argparse.ArgumentParser(prog="pace_info", + description="Utility to analyze PACE (.yaml) potential shape and other parameters") -parser.add_argument("potential_file", help="B-basis file name (.yaml)", type=str, nargs='+', default=[]) + parser.add_argument("potential_file", help="B-basis file name (.yaml)", type=str, nargs='+', default=[]) -args_parse = parser.parse_args() -potential_files = args_parse.potential_file + args_parse = parser.parse_args() + potential_files = args_parse.potential_file -for potential_file in potential_files: - # print("potential file: ", potential_file) - analyse_potential_shape(potential_file) + for potential_file in potential_files: + # print("potential file: ", potential_file) + analyse_potential_shape(potential_file) + + +if __name__ == "__main__": + main() diff --git a/bin/pace_select b/src/pyace/cli/pace_select.py similarity index 99% rename from bin/pace_select rename to src/pyace/cli/pace_select.py index a572967..3fba20c 100644 --- a/bin/pace_select +++ b/src/pyace/cli/pace_select.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python import argparse import logging diff --git a/src/pyace/cli/pace_timing.py b/src/pyace/cli/pace_timing.py new file mode 100644 index 0000000..e7a42a5 --- /dev/null +++ b/src/pyace/cli/pace_timing.py @@ -0,0 +1,28 @@ +import argparse +import numpy as np + +from pyace.utils.timing import run_timing_test + + +def main(): + parser = argparse.ArgumentParser(prog="pace_timing", + description="Utility to run the single-CPU timing test for PACE (.yaml) potential ") + + parser.add_argument("potential_file", help="B-basis file name (.yaml)", type=str, nargs='+', default=[]) + parser.add_argument("-n", "--nstructures", help="number of structures", type=int, default=10) + + args_parse = parser.parse_args() + potential_files = args_parse.potential_file + nstructures = args_parse.nstructures + + for potential_file in potential_files: + np.random.seed(42) + print() + print("**************************************************") + print("Using potential file: ", potential_file) + print("**************************************************") + run_timing_test(potential_file, n_struct=nstructures) + + +if __name__ == "__main__": + main() diff --git a/src/pyace/cli/pace_yaml2yace.py b/src/pyace/cli/pace_yaml2yace.py new file mode 100644 index 0000000..d0d3d13 --- /dev/null +++ b/src/pyace/cli/pace_yaml2yace.py @@ -0,0 +1,37 @@ +import argparse + +from pyace import ACEBBasisSet + + +def main(): + parser = argparse.ArgumentParser(prog="pace_yaml2yace", + description="Conversion utility from B-basis (.yaml file) to new-style Ctilde-basis (.yace file)") + parser.add_argument("input", help="input B-basis file name (.yaml)", type=str, nargs='+', default=[]) + parser.add_argument("-o", "--output", help="output Ctilde-basis file name (.yace)", type=str, default="") + + args_parse = parser.parse_args() + input_yaml_filenames = args_parse.input + output_ace_filename = args_parse.output + + + for input_yaml_filename in input_yaml_filenames: + if output_ace_filename == "": + if input_yaml_filename.endswith("yaml"): + actual_output_ace_filename = input_yaml_filename.replace("yaml", "yace") + elif input_yaml_filename.endswith("yml"): + actual_output_ace_filename = input_yaml_filename.replace("yml", "yace") + else: + actual_output_ace_filename = input_yaml_filename + ".yace" + else: + actual_output_ace_filename = output_ace_filename + + print("Loading B-basis from '{}'".format(input_yaml_filename)) + bbasis = ACEBBasisSet(input_yaml_filename) + print("Converting to Ctilde-basis") + cbasis = bbasis.to_ACECTildeBasisSet() + print("Saving Ctilde-basis to '{}'".format(actual_output_ace_filename)) + cbasis.save_yaml(actual_output_ace_filename) + + +if __name__ == "__main__": + main() diff --git a/bin/pacemaker b/src/pyace/cli/pacemaker.py similarity index 99% rename from bin/pacemaker rename to src/pyace/cli/pacemaker.py index 4736d94..2a939c0 100644 --- a/bin/pacemaker +++ b/src/pyace/cli/pacemaker.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python import argparse import getpass # for getpass.getuser() @@ -48,7 +47,7 @@ DEFAULT_SEED = 42 -def main(args): +def main(args=None): parser = argparse.ArgumentParser(prog="pacemaker", description="Fitting utility for atomic cluster expansion " "potentials.\n" + "version: {}".format(__version__)) @@ -397,4 +396,4 @@ def predict_and_save(general_fit, target_bbasisconfig, structures_dataframe, fna if __name__ == "__main__": - main(sys.argv[1:]) + main() From 8dbc3d15d645ba0f8a18a299ca5ee8c42f45d4e1 Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Fri, 4 Sep 2026 14:46:28 +0200 Subject: [PATCH 14/30] build: drop python upper bound, matching grace-tensorpotential requires-python is now ">=3.9", identical to grace-tensorpotential's. The "<3.14" added in 17aba14 was derived from tensorflow<=2.20 shipping wheels only up to 3.13, but an upper bound on requires-python is resolver metadata that cannot be corrected after release: once tensorflow adds 3.14, every already-published pyace would stay unselectable on it. The constraint is also redundant -- pip cannot install tensorflow on a python it has no wheels for, so a grace environment is limited whether or not pyace repeats the limit. --- pyproject.toml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9896b2c..ad6b269 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,11 +29,12 @@ authors = [ ] description = 'Python bindings, utilities for PACE and fitting code "pacemaker"' readme = "README.md" -# Upper bound tracks grace-tensorpotential, which pyace must stay installable -# alongside. It declares only ">=3.9", so the real ceiling is its tensorflow<=2.20 -# dependency: tensorflow 2.20 publishes wheels for 3.9-3.13 and nothing newer, -# so a grace environment cannot exist on 3.14. Revisit when tensorflow adds it. -requires-python = ">=3.9,<3.14" +# Matches grace-tensorpotential, which pyace must stay installable beside. +# Deliberately no upper bound: a ceiling here would apply to the resolver +# forever, making an already-published release unselectable on a python it +# would actually work on. Where the limit is real it is enforced already -- +# tensorflow publishes no wheels for unsupported pythons. +requires-python = ">=3.9" license = { text = "Academic Software License (ASL)" } classifiers = [ "License :: Other/Proprietary License", From 89b9ca4dbb37f4a7a987f72e60bb8975f0986ed0 Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Fri, 4 Sep 2026 15:14:55 +0200 Subject: [PATCH 15/30] build: clear the warnings a verbose build was emitting Four classes of warning, all from code added on this branch. Verified by rebuilding with CMAKE_ARGS=-DCMAKE_VERBOSE_MAKEFILE=ON and diffing counts. numpy C-API (2 -> 0) The vendored maxvol kernel compiled against the pre-1.7 numpy API, so every build printed numpy's "Using deprecated NumPy API" #warning. Define NPY_NO_DEPRECATED_API; it compiles clean, no source change needed. setuptools / PEP 639 (9 -> 0) `project.license` as a TOML table and "License ::" classifiers are both deprecated, and setuptools warned six and three times respectively. Replace with an SPDX expression plus license-files, which also ships the vendored MIT notice as declared license metadata rather than only as package data. Needs setuptools>=77, hence the build-requires bump. Note the deprecated form was copied from grace-tensorpotential, which still has it. uninitialised pivot indices (0 -> 0, exposed by the above) Defining NPY_NO_DEPRECATED_API changed Cython's codegen enough for gcc to notice that dmaxvol's i and j are only assigned inside the max-search loop. For a zero-column matrix that loop never runs and both were read uninitialised -- present in maxvolpy too. Initialise them and reject the degenerate shape up front. build-time numpy cap numpy>=2.0 let pip pick any future major. The 2.x C-API is frozen so a wheel built against 2.2.6 imports fine under 2.1.3 (verified), but a numpy 3.0 ABI break would silently raise the built wheel's runtime floor above the numpy>=2.0 we advertise. Cap the build pin at <3. Left alone deliberately: four CMake cmake_minimum_required(3.7) deprecations and CMP0148, which come from CMakeLists.txt plus vendored pybind11, and seven gcc LTO "serial compilation of N LTRANS jobs" notices. --- pyproject.toml | 18 +++++++++++------- setup.py | 3 +++ src/pyace/maxvol/_maxvol.pyx | 7 ++++++- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ad6b269..0aae8f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,16 +4,18 @@ # make, which schedules this project's seven modules much less effectively, and # without Cython the maxvol extension is skipped entirely. # -# Build against numpy>=2.0 for pyace.maxvol._maxvol: a binary compiled against -# the 2.x C-API also runs under numpy 1.19+, so this is the widest-compatibility -# choice rather than a floor imposed on users. +# Build against numpy>=2.0 for pyace.maxvol._maxvol: the 2.x C-API is frozen, so +# a binary compiled against any 2.x runs under every numpy >= 2.0 (verified: +# built against 2.2.6, imports fine under 2.1.3). Capped below 3.0 because the +# next ABI break would silently raise the built wheel's runtime floor above the +# numpy>=2.0 we advertise in [project.dependencies]. requires = [ - "setuptools>=64", + "setuptools>=77", # PEP 639 license/license-files support "wheel", "cmake>=3.15", "ninja; platform_system!='Windows'", "Cython>=0.29.32", - "numpy>=2.0", + "numpy>=2.0,<3", "scipy", ] build-backend = "setuptools.build_meta" @@ -35,9 +37,11 @@ readme = "README.md" # would actually work on. Where the limit is real it is enforced already -- # tensorflow publishes no wheels for unsupported pythons. requires-python = ">=3.9" -license = { text = "Academic Software License (ASL)" } +license = "LicenseRef-Academic-Software-License" +license-files = ["LICENSE.md", "src/pyace/maxvol/LICENSE.maxvolpy.txt"] classifiers = [ - "License :: Other/Proprietary License", + # No "License ::" classifier: PEP 639 deprecates them in favour of the + # SPDX expression above. "Programming Language :: Python :: 3", ] dependencies = [ diff --git a/setup.py b/setup.py index ab2a626..b8a96d9 100644 --- a/setup.py +++ b/setup.py @@ -230,6 +230,9 @@ def maxvol_extension(): "pyace.maxvol._maxvol", ["src/pyace/maxvol/_maxvol.pyx"], include_dirs=[numpy.get_include()], + # Compile against the modern numpy C-API. Without this every build + # prints numpy's "Using deprecated NumPy API" #warning. + define_macros=[("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION")], # No -march=native: it bakes in the build machine's ISA and # SIGILLs on any older CPU, which makes wheels non-redistributable. # No -ffast-math either -- this is a pivoting algorithm whose diff --git a/src/pyace/maxvol/_maxvol.pyx b/src/pyace/maxvol/_maxvol.pyx index 485c3b2..6e605d7 100644 --- a/src/pyace/maxvol/_maxvol.pyx +++ b/src/pyace/maxvol/_maxvol.pyx @@ -15,6 +15,9 @@ Two deliberate differences from the original: swaps wrote -- and it was only ever safe because nothing passed -fopenmp. * the scratch buffers are released in a ``finally``, so an error raised by dgetrf no longer leaks them. +* the pivot indices are initialised, and a zero-column matrix is rejected up + front; previously both were only assigned inside a loop the compiler cannot + prove executes, which for R == 0 meant reading them uninitialised. """ __all__ = ['c_maxvol'] @@ -43,6 +46,8 @@ def c_maxvol(A, tol=1.05, max_iters=100, top_k_index=-1, int verbose=0): raise TypeError("argument must be of numpy.ndarray type") if A.ndim != 2: raise ValueError("argument must have 2 dimensions") + if A.shape[1] < 1: + raise ValueError("argument must have at least one column") if A.dtype != np.dtype(np.float64): raise TypeError("argument must be of float64 type, got {}".format(A.dtype)) N, r = A.shape @@ -64,7 +69,7 @@ cdef object dmaxvol(int N, int R, double *lu, double *coef, int *basis, cdef int *interchange = malloc(N * sizeof(int)) cdef double *tmp_row = malloc(R * sizeof(double)) cdef double *tmp_column = malloc(N * sizeof(double)) - cdef int info = 0, i, j, tmp_int, i_one = 1, iters = 0 + cdef int info = 0, i = 0, j = 0, tmp_int, i_one = 1, iters = 0 cdef int k_row, k_col cdef char cR = b'R', cN = b'N', cU = b'U', cL = b'L' cdef double d_one = 1, alpha, max_value From 7148751514037e1de87327d05888c980bd4fdcfc Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Fri, 4 Sep 2026 15:33:37 +0200 Subject: [PATCH 16/30] build: upgrade vendored pybind11 2.11.0.dev1 -> 3.1.0 The vendored copy was an unreleased dev snapshot (2.11.0.dev1) whose CMake declared cmake_minimum_required(VERSION 3.7) and used FindPythonLibsNew, producing a CMake deprecation warning plus a CMP0148 policy warning on CMake 4 -- both of which become hard errors in a future CMake. 2.13.6, the last 2.x, still declares 3.5, so only the 3.x line actually fixes this: 3.1.0 declares cmake_minimum_required(VERSION 3.15...4.2). Checked before committing to it: pybind11 3.x does not require C++17. Its common.h only hard-errors on Python < 3.8 and old compilers, and its CMakeLists respects an externally set CMAKE_CXX_STANDARD, so CMAKE_CXX_STANDARD 11 still works and no ACE source needed touching. 3.1.0's requires-python is >=3.9, matching ours. Our side needed two changes: - cmake_minimum_required 3.7 -> 3.15 in CMakeLists.txt, lib/ace/CMakeLists.txt and lib/ace/ace-evaluator/CMakeLists.txt. 3.15 is pybind11 3.x's floor, and this clears the remaining three deprecation warnings. Plain 3.15 rather than a range, to take 3.15 policy defaults rather than newer ones. - setup.py passes -DPython_EXECUTABLE alongside -DPYTHON_EXECUTABLE, since 3.x uses FindPython (Python_EXECUTABLE) rather than FindPythonLibsNew. Vendoring reduced to what the build consumes: CMakeLists.txt, include/, tools/, pybind11/, LICENSE, README.rst, pyproject.toml. The old copy also carried upstream's CI and lint config (.appveyor.yml, .clang-tidy, .cmake-format.yaml, .readthedocs.yml, noxfield/setup.py/setup.cfg/MANIFEST.in); most no longer exist in 3.1.0 and none had a role here. BSD-3-Clause LICENSE retained. Verified: configure emits zero CMake warnings (was six), build has zero compiler warnings, all 7 modules plus the maxvol kernel import, and 292 tests pass. --- CMakeLists.txt | 2 +- lib/ace/CMakeLists.txt | 2 +- lib/ace/ace-evaluator/CMakeLists.txt | 2 +- lib/pybind11/.appveyor.yml | 35 - lib/pybind11/.clang-format | 38 - lib/pybind11/.clang-tidy | 77 - lib/pybind11/.cmake-format.yaml | 73 - lib/pybind11/.codespell-ignore-lines | 24 - lib/pybind11/.gitattributes | 1 - lib/pybind11/.gitignore | 46 - lib/pybind11/.readthedocs.yml | 3 - lib/pybind11/CMakeLists.txt | 222 +- lib/pybind11/MANIFEST.in | 5 - lib/pybind11/README.rst | 85 +- lib/pybind11/include/pybind11/attr.h | 78 +- lib/pybind11/include/pybind11/buffer_info.h | 52 +- lib/pybind11/include/pybind11/cast.h | 1103 +++++++-- lib/pybind11/include/pybind11/chrono.h | 4 +- lib/pybind11/include/pybind11/complex.h | 29 +- .../include/pybind11/conduit/README.txt | 15 + .../pybind11/conduit/pybind11_conduit_v1.h | 116 + .../conduit/pybind11_platform_abi_id.h | 87 + .../pybind11/conduit/wrap_include_python_h.h | 72 + .../include/pybind11/critical_section.h | 56 + .../include/pybind11/detail/argument_vector.h | 417 ++++ lib/pybind11/include/pybind11/detail/class.h | 365 ++- lib/pybind11/include/pybind11/detail/common.h | 623 +++-- .../include/pybind11/detail/cpp_conduit.h | 75 + lib/pybind11/include/pybind11/detail/descr.h | 73 + .../detail/dynamic_raw_ptr_cast_if_possible.h | 39 + .../pybind11/detail/exception_translation.h | 71 + .../detail/function_record_pyobject.h | 191 ++ .../include/pybind11/detail/function_ref.h | 101 + .../detail/holder_caster_foreign_helpers.h | 104 + lib/pybind11/include/pybind11/detail/init.h | 176 +- .../include/pybind11/detail/internals.h | 1088 +++++--- .../pybind11/detail/native_enum_data.h | 227 ++ .../detail/pybind11_namespace_macros.h | 82 + .../pybind11/detail/struct_smart_holder.h | 398 +++ .../pybind11/detail/type_caster_base.h | 1197 +++++++-- .../pybind11/detail/using_smart_holder.h | 22 + .../pybind11/detail/value_and_holder.h | 92 + lib/pybind11/include/pybind11/eigen/common.h | 9 + lib/pybind11/include/pybind11/eigen/matrix.h | 44 +- lib/pybind11/include/pybind11/eigen/tensor.h | 35 +- lib/pybind11/include/pybind11/embed.h | 175 +- lib/pybind11/include/pybind11/eval.h | 22 +- lib/pybind11/include/pybind11/functional.h | 99 +- lib/pybind11/include/pybind11/gil.h | 122 +- .../include/pybind11/gil_safe_call_once.h | 283 +++ lib/pybind11/include/pybind11/gil_simple.h | 37 + lib/pybind11/include/pybind11/iostream.h | 44 +- lib/pybind11/include/pybind11/native_enum.h | 76 + lib/pybind11/include/pybind11/numpy.h | 592 ++++- lib/pybind11/include/pybind11/pybind11.h | 2181 +++++++++++++---- lib/pybind11/include/pybind11/pytypes.h | 405 ++- lib/pybind11/include/pybind11/stl.h | 333 ++- .../include/pybind11/stl/filesystem.h | 48 +- lib/pybind11/include/pybind11/stl_bind.h | 165 +- .../include/pybind11/subinterpreter.h | 465 ++++ .../pybind11/trampoline_self_life_support.h | 65 + .../pybind11/type_caster_pyobject_ptr.h | 61 + lib/pybind11/include/pybind11/typing.h | 293 +++ lib/pybind11/include/pybind11/warnings.h | 75 + lib/pybind11/noxfile.py | 107 - lib/pybind11/pybind11/__init__.py | 10 +- lib/pybind11/pybind11/__main__.py | 91 +- lib/pybind11/pybind11/_version.py | 28 +- lib/pybind11/pybind11/commands.py | 105 +- lib/pybind11/pybind11/setup_helpers.py | 97 +- lib/pybind11/pyproject.toml | 232 +- lib/pybind11/setup.cfg | 50 - lib/pybind11/setup.py | 150 -- lib/pybind11/tools/FindCatch.cmake | 10 +- lib/pybind11/tools/FindPythonLibsNew.cmake | 39 +- .../codespell_ignore_lines_from_errors.py | 13 +- lib/pybind11/tools/libsize.py | 2 + lib/pybind11/tools/make_changelog.py | 140 +- lib/pybind11/tools/make_global.py | 33 + lib/pybind11/tools/pybind11Common.cmake | 227 +- lib/pybind11/tools/pybind11Config.cmake.in | 19 +- .../tools/pybind11GuessPythonExtSuffix.cmake | 94 + lib/pybind11/tools/pybind11NewTools.cmake | 214 +- lib/pybind11/tools/pybind11Tools.cmake | 76 +- lib/pybind11/tools/pyproject.toml | 3 - lib/pybind11/tools/setup_global.py.in | 63 - lib/pybind11/tools/setup_main.py.in | 44 - .../test-pybind11GuessPythonExtSuffix.cmake | 185 ++ setup.py | 3 + 89 files changed, 11849 insertions(+), 3383 deletions(-) delete mode 100644 lib/pybind11/.appveyor.yml delete mode 100644 lib/pybind11/.clang-format delete mode 100644 lib/pybind11/.clang-tidy delete mode 100644 lib/pybind11/.cmake-format.yaml delete mode 100644 lib/pybind11/.codespell-ignore-lines delete mode 100644 lib/pybind11/.gitattributes delete mode 100644 lib/pybind11/.gitignore delete mode 100644 lib/pybind11/.readthedocs.yml delete mode 100644 lib/pybind11/MANIFEST.in create mode 100644 lib/pybind11/include/pybind11/conduit/README.txt create mode 100644 lib/pybind11/include/pybind11/conduit/pybind11_conduit_v1.h create mode 100644 lib/pybind11/include/pybind11/conduit/pybind11_platform_abi_id.h create mode 100644 lib/pybind11/include/pybind11/conduit/wrap_include_python_h.h create mode 100644 lib/pybind11/include/pybind11/critical_section.h create mode 100644 lib/pybind11/include/pybind11/detail/argument_vector.h create mode 100644 lib/pybind11/include/pybind11/detail/cpp_conduit.h create mode 100644 lib/pybind11/include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h create mode 100644 lib/pybind11/include/pybind11/detail/exception_translation.h create mode 100644 lib/pybind11/include/pybind11/detail/function_record_pyobject.h create mode 100644 lib/pybind11/include/pybind11/detail/function_ref.h create mode 100644 lib/pybind11/include/pybind11/detail/holder_caster_foreign_helpers.h create mode 100644 lib/pybind11/include/pybind11/detail/native_enum_data.h create mode 100644 lib/pybind11/include/pybind11/detail/pybind11_namespace_macros.h create mode 100644 lib/pybind11/include/pybind11/detail/struct_smart_holder.h create mode 100644 lib/pybind11/include/pybind11/detail/using_smart_holder.h create mode 100644 lib/pybind11/include/pybind11/detail/value_and_holder.h create mode 100644 lib/pybind11/include/pybind11/eigen/common.h create mode 100644 lib/pybind11/include/pybind11/gil_safe_call_once.h create mode 100644 lib/pybind11/include/pybind11/gil_simple.h create mode 100644 lib/pybind11/include/pybind11/native_enum.h create mode 100644 lib/pybind11/include/pybind11/subinterpreter.h create mode 100644 lib/pybind11/include/pybind11/trampoline_self_life_support.h create mode 100644 lib/pybind11/include/pybind11/type_caster_pyobject_ptr.h create mode 100644 lib/pybind11/include/pybind11/typing.h create mode 100644 lib/pybind11/include/pybind11/warnings.h delete mode 100644 lib/pybind11/noxfile.py delete mode 100644 lib/pybind11/setup.cfg delete mode 100644 lib/pybind11/setup.py create mode 100755 lib/pybind11/tools/make_global.py create mode 100644 lib/pybind11/tools/pybind11GuessPythonExtSuffix.cmake delete mode 100644 lib/pybind11/tools/pyproject.toml delete mode 100644 lib/pybind11/tools/setup_global.py.in delete mode 100644 lib/pybind11/tools/setup_main.py.in create mode 100644 lib/pybind11/tools/test-pybind11GuessPythonExtSuffix.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 30011ed..25396c3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.7) +cmake_minimum_required(VERSION 3.15) find_program(CMAKE_C_COMPILER NAMES $ENV{CC} gcc PATHS ENV PATH NO_DEFAULT_PATH) find_program(CMAKE_CXX_COMPILER NAMES $ENV{CXX} g++ PATHS ENV PATH NO_DEFAULT_PATH) diff --git a/lib/ace/CMakeLists.txt b/lib/ace/CMakeLists.txt index 3a5f452..5428c85 100644 --- a/lib/ace/CMakeLists.txt +++ b/lib/ace/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.7) # CMake version check +cmake_minimum_required(VERSION 3.15) # CMake version check project(ace) set(CMAKE_CXX_STANDARD 11) # Enable c++11 standard set(CMAKE_POSITION_INDEPENDENT_CODE ON) diff --git a/lib/ace/ace-evaluator/CMakeLists.txt b/lib/ace/ace-evaluator/CMakeLists.txt index aa3576c..6b95448 100644 --- a/lib/ace/ace-evaluator/CMakeLists.txt +++ b/lib/ace/ace-evaluator/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.7) # CMake version check +cmake_minimum_required(VERSION 3.15) # CMake version check project(ace-evaluator) set(CMAKE_CXX_STANDARD 11) # Enable c++11 standard set(CMAKE_POSITION_INDEPENDENT_CODE ON) diff --git a/lib/pybind11/.appveyor.yml b/lib/pybind11/.appveyor.yml deleted file mode 100644 index 360760a..0000000 --- a/lib/pybind11/.appveyor.yml +++ /dev/null @@ -1,35 +0,0 @@ -version: 1.0.{build} -image: -- Visual Studio 2017 -test: off -skip_branch_with_pr: true -build: - parallel: true -platform: -- x86 -environment: - matrix: - - PYTHON: 36 - CONFIG: Debug -install: -- ps: | - $env:CMAKE_GENERATOR = "Visual Studio 15 2017" - if ($env:PLATFORM -eq "x64") { $env:PYTHON = "$env:PYTHON-x64" } - $env:PATH = "C:\Python$env:PYTHON\;C:\Python$env:PYTHON\Scripts\;$env:PATH" - python -W ignore -m pip install --upgrade pip wheel - python -W ignore -m pip install pytest numpy --no-warn-script-location pytest-timeout -- ps: | - Start-FileDownload 'https://gitlab.com/libeigen/eigen/-/archive/3.3.7/eigen-3.3.7.zip' - 7z x eigen-3.3.7.zip -y > $null - $env:CMAKE_INCLUDE_PATH = "eigen-3.3.7;$env:CMAKE_INCLUDE_PATH" -build_script: -- cmake -G "%CMAKE_GENERATOR%" -A "%CMAKE_ARCH%" - -DCMAKE_CXX_STANDARD=14 - -DPYBIND11_WERROR=ON - -DDOWNLOAD_CATCH=ON - -DCMAKE_SUPPRESS_REGENERATION=1 - . -- set MSBuildLogger="C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" -- cmake --build . --config %CONFIG% --target pytest -- /m /v:m /logger:%MSBuildLogger% -- cmake --build . --config %CONFIG% --target cpptest -- /m /v:m /logger:%MSBuildLogger% -on_failure: if exist "tests\test_cmake_build" type tests\test_cmake_build\*.log* diff --git a/lib/pybind11/.clang-format b/lib/pybind11/.clang-format deleted file mode 100644 index b477a16..0000000 --- a/lib/pybind11/.clang-format +++ /dev/null @@ -1,38 +0,0 @@ ---- -# See all possible options and defaults with: -# clang-format --style=llvm --dump-config -BasedOnStyle: LLVM -AccessModifierOffset: -4 -AllowShortLambdasOnASingleLine: true -AlwaysBreakTemplateDeclarations: Yes -BinPackArguments: false -BinPackParameters: false -BreakBeforeBinaryOperators: All -BreakConstructorInitializers: BeforeColon -ColumnLimit: 99 -CommentPragmas: 'NOLINT:.*|^ IWYU pragma:' -IncludeBlocks: Regroup -IndentCaseLabels: true -IndentPPDirectives: AfterHash -IndentWidth: 4 -Language: Cpp -SpaceAfterCStyleCast: true -Standard: Cpp11 -StatementMacros: ['PyObject_HEAD'] -TabWidth: 4 -IncludeCategories: - - Regex: '' - Priority: 4 - - Regex: '.*' - Priority: 5 -... diff --git a/lib/pybind11/.clang-tidy b/lib/pybind11/.clang-tidy deleted file mode 100644 index 2301838..0000000 --- a/lib/pybind11/.clang-tidy +++ /dev/null @@ -1,77 +0,0 @@ -FormatStyle: file - -Checks: | - *bugprone*, - *performance*, - clang-analyzer-optin.cplusplus.VirtualCall, - clang-analyzer-optin.performance.Padding, - cppcoreguidelines-init-variables, - cppcoreguidelines-prefer-member-initializer, - cppcoreguidelines-pro-type-static-cast-downcast, - cppcoreguidelines-slicing, - google-explicit-constructor, - llvm-namespace-comment, - misc-definitions-in-headers, - misc-misplaced-const, - misc-non-copyable-objects, - misc-static-assert, - misc-throw-by-value-catch-by-reference, - misc-uniqueptr-reset-release, - misc-unused-parameters, - modernize-avoid-bind, - modernize-loop-convert, - modernize-make-shared, - modernize-redundant-void-arg, - modernize-replace-auto-ptr, - modernize-replace-disallow-copy-and-assign-macro, - modernize-replace-random-shuffle, - modernize-shrink-to-fit, - modernize-use-auto, - modernize-use-bool-literals, - modernize-use-default-member-init, - modernize-use-emplace, - modernize-use-equals-default, - modernize-use-equals-delete, - modernize-use-noexcept, - modernize-use-nullptr, - modernize-use-override, - modernize-use-using, - readability-avoid-const-params-in-decls, - readability-braces-around-statements, - readability-const-return-type, - readability-container-size-empty, - readability-delete-null-pointer, - readability-else-after-return, - readability-implicit-bool-conversion, - readability-inconsistent-declaration-parameter-name, - readability-make-member-function-const, - readability-misplaced-array-index, - readability-non-const-parameter, - readability-qualified-auto, - readability-redundant-function-ptr-dereference, - readability-redundant-smartptr-get, - readability-redundant-string-cstr, - readability-simplify-subscript-expr, - readability-static-accessed-through-instance, - readability-static-definition-in-anonymous-namespace, - readability-string-compare, - readability-suspicious-call-argument, - readability-uniqueptr-delete-release, - -bugprone-easily-swappable-parameters, - -bugprone-exception-escape, - -bugprone-reserved-identifier, - -bugprone-unused-raii, - -CheckOptions: -- key: modernize-use-equals-default.IgnoreMacros - value: false -- key: performance-for-range-copy.WarnOnAllAutoCopies - value: true -- key: performance-inefficient-string-concatenation.StrictMode - value: true -- key: performance-unnecessary-value-param.AllowedTypes - value: 'exception_ptr$;' -- key: readability-implicit-bool-conversion.AllowPointerConditions - value: true - -HeaderFilterRegex: 'pybind11/.*h' diff --git a/lib/pybind11/.cmake-format.yaml b/lib/pybind11/.cmake-format.yaml deleted file mode 100644 index a2a69f3..0000000 --- a/lib/pybind11/.cmake-format.yaml +++ /dev/null @@ -1,73 +0,0 @@ -parse: - additional_commands: - pybind11_add_module: - flags: - - THIN_LTO - - MODULE - - SHARED - - NO_EXTRAS - - EXCLUDE_FROM_ALL - - SYSTEM - -format: - line_width: 99 - tab_size: 2 - - # If an argument group contains more than this many sub-groups - # (parg or kwarg groups) then force it to a vertical layout. - max_subgroups_hwrap: 2 - - # If a positional argument group contains more than this many - # arguments, then force it to a vertical layout. - max_pargs_hwrap: 6 - - # If a cmdline positional group consumes more than this many - # lines without nesting, then invalidate the layout (and nest) - max_rows_cmdline: 2 - separate_ctrl_name_with_space: false - separate_fn_name_with_space: false - dangle_parens: false - - # If the trailing parenthesis must be 'dangled' on its on - # 'line, then align it to this reference: `prefix`: the start' - # 'of the statement, `prefix-indent`: the start of the' - # 'statement, plus one indentation level, `child`: align to' - # the column of the arguments - dangle_align: prefix - # If the statement spelling length (including space and - # parenthesis) is smaller than this amount, then force reject - # nested layouts. - min_prefix_chars: 4 - - # If the statement spelling length (including space and - # parenthesis) is larger than the tab width by more than this - # amount, then force reject un-nested layouts. - max_prefix_chars: 10 - - # If a candidate layout is wrapped horizontally but it exceeds - # this many lines, then reject the layout. - max_lines_hwrap: 2 - - line_ending: unix - - # Format command names consistently as 'lower' or 'upper' case - command_case: canonical - - # Format keywords consistently as 'lower' or 'upper' case - # unchanged is valid too - keyword_case: 'upper' - - # A list of command names which should always be wrapped - always_wrap: [] - - # If true, the argument lists which are known to be sortable - # will be sorted lexicographically - enable_sort: true - - # If true, the parsers may infer whether or not an argument - # list is sortable (without annotation). - autosort: false - -# Causes a few issues - can be solved later, possibly. -markup: - enable_markup: false diff --git a/lib/pybind11/.codespell-ignore-lines b/lib/pybind11/.codespell-ignore-lines deleted file mode 100644 index 2a01d63..0000000 --- a/lib/pybind11/.codespell-ignore-lines +++ /dev/null @@ -1,24 +0,0 @@ -template - template - auto &this_ = static_cast(*this); - if (load_impl(temp, false)) { - ssize_t nd = 0; - auto trivial = broadcast(buffers, nd, shape); - auto ndim = (size_t) nd; - int nd; - ssize_t ndim() const { return detail::array_proxy(m_ptr)->nd; } - using op = op_impl; -template - template - class_ &def(const detail::op_ &op, const Extra &...extra) { - class_ &def_cast(const detail::op_ &op, const Extra &...extra) { -@pytest.mark.parametrize("access", ["ro", "rw", "static_ro", "static_rw"]) -struct IntStruct { - explicit IntStruct(int v) : value(v){}; - ~IntStruct() { value = -value; } - IntStruct(const IntStruct &) = default; - IntStruct &operator=(const IntStruct &) = default; - py::class_(m, "IntStruct").def(py::init([](const int i) { return IntStruct(i); })); - py::implicitly_convertible(); - m.def("test", [](int expected, const IntStruct &in) { - [](int expected, const IntStruct &in) { diff --git a/lib/pybind11/.gitattributes b/lib/pybind11/.gitattributes deleted file mode 100644 index d611e14..0000000 --- a/lib/pybind11/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -docs/*.svg binary diff --git a/lib/pybind11/.gitignore b/lib/pybind11/.gitignore deleted file mode 100644 index 43d5094..0000000 --- a/lib/pybind11/.gitignore +++ /dev/null @@ -1,46 +0,0 @@ -CMakeCache.txt -CMakeFiles -Makefile -cmake_install.cmake -cmake_uninstall.cmake -.DS_Store -*.so -*.pyd -*.dll -*.sln -*.sdf -*.opensdf -*.vcxproj -*.vcxproj.user -*.filters -example.dir -Win32 -x64 -Release -Debug -.vs -CTestTestfile.cmake -Testing -autogen -MANIFEST -/.ninja_* -/*.ninja -/docs/.build -*.py[co] -*.egg-info -*~ -.*.swp -.DS_Store -/dist -/*build* -.cache/ -sosize-*.txt -pybind11Config*.cmake -pybind11Targets.cmake -/*env* -/.vscode -/pybind11/include/* -/pybind11/share/* -/docs/_build/* -.ipynb_checkpoints/ -tests/main.cpp diff --git a/lib/pybind11/.readthedocs.yml b/lib/pybind11/.readthedocs.yml deleted file mode 100644 index c9c6161..0000000 --- a/lib/pybind11/.readthedocs.yml +++ /dev/null @@ -1,3 +0,0 @@ -python: - version: 3 -requirements_file: docs/requirements.txt diff --git a/lib/pybind11/CMakeLists.txt b/lib/pybind11/CMakeLists.txt index 229658e..6f3f8be 100644 --- a/lib/pybind11/CMakeLists.txt +++ b/lib/pybind11/CMakeLists.txt @@ -5,21 +5,20 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. -cmake_minimum_required(VERSION 3.7) +# Propagate this policy (FindPythonInterp removal) so it can be detected later +if(NOT CMAKE_VERSION VERSION_LESS "3.27") + cmake_policy(GET CMP0148 _pybind11_cmp0148) +endif() -# The `cmake_minimum_required(VERSION 3.4...3.22)` syntax does not work with -# some versions of VS that have a patched CMake 3.11. This forces us to emulate -# the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.22) - cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) -else() - cmake_policy(VERSION 3.22) +cmake_minimum_required(VERSION 3.15...4.2) + +if(_pybind11_cmp0148) + cmake_policy(SET CMP0148 ${_pybind11_cmp0148}) + unset(_pybind11_cmp0148) endif() # Avoid infinite recursion if tests include this as a subdirectory -if(DEFINED PYBIND11_MASTER_PROJECT) - return() -endif() +include_guard(GLOBAL) # Extract project version from source file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/include/pybind11/detail/common.h" @@ -64,16 +63,15 @@ if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) set(PYBIND11_MASTER_PROJECT ON) - if(OSX AND CMAKE_VERSION VERSION_LESS 3.7) - # Bug in macOS CMake < 3.7 is unable to download catch - message(WARNING "CMAKE 3.7+ needed on macOS to download catch, and newer HIGHLY recommended") - elseif(WINDOWS AND CMAKE_VERSION VERSION_LESS 3.8) - # Only tested with 3.8+ in CI. - message(WARNING "CMAKE 3.8+ tested on Windows, previous versions untested") - endif() - message(STATUS "CMake ${CMAKE_VERSION}") + if(DEFINED SKBUILD AND DEFINED ENV{PYBIND11_GLOBAL_SDIST}) + message( + FATAL_ERROR + "PYBIND11_GLOBAL_SDIST is not supported, use nox -s build_global or a pybind11-global SDist instead." + ) + endif() + if(CMAKE_CXX_STANDARD) set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -82,21 +80,38 @@ if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) set(pybind11_system "") set_property(GLOBAL PROPERTY USE_FOLDERS ON) + if(CMAKE_VERSION VERSION_LESS "3.18") + set(_pybind11_findpython_default OFF) + else() + set(_pybind11_findpython_default ON) + endif() else() set(PYBIND11_MASTER_PROJECT OFF) set(pybind11_system SYSTEM) + set(_pybind11_findpython_default COMPAT) endif() # Options option(PYBIND11_INSTALL "Install pybind11 header files?" ${PYBIND11_MASTER_PROJECT}) option(PYBIND11_TEST "Build pybind11 test suite?" ${PYBIND11_MASTER_PROJECT}) option(PYBIND11_NOPYTHON "Disable search for Python" OFF) +option(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION + "To enforce that a handle_type_name<> specialization exists" OFF) option(PYBIND11_SIMPLE_GIL_MANAGEMENT "Use simpler GIL management logic that does not support disassociation" OFF) set(PYBIND11_INTERNALS_VERSION "" CACHE STRING "Override the ABI version, may be used to enable the unstable ABI.") +# Record whether the project set this before option() supplies its OFF default, so +# pybind11Common.cmake can apply its CMP0190-aware default without clobbering an explicit value. +if(NOT DEFINED PYBIND11_USE_CROSSCOMPILING) + set(_PYBIND11_USE_CROSSCOMPILING_DEFAULTED ON) +endif() +option(PYBIND11_USE_CROSSCOMPILING "Respect CMAKE_CROSSCOMPILING" OFF) +if(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION) + add_compile_definitions(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION) +endif() if(PYBIND11_SIMPLE_GIL_MANAGEMENT) add_compile_definitions(PYBIND11_SIMPLE_GIL_MANAGEMENT) endif() @@ -106,43 +121,138 @@ cmake_dependent_option( "Install pybind11 headers in Python include directory instead of default installation prefix" OFF "PYBIND11_INSTALL" OFF) -cmake_dependent_option(PYBIND11_FINDPYTHON "Force new FindPython" OFF - "NOT CMAKE_VERSION VERSION_LESS 3.12" OFF) +set(PYBIND11_FINDPYTHON + ${_pybind11_findpython_default} + CACHE STRING "Force new FindPython - NEW, OLD, COMPAT") + +if(PYBIND11_MASTER_PROJECT) + + # Allow PYTHON_EXECUTABLE if in FINDPYTHON mode and building pybind11's tests + # (makes transition easier while we support both modes). + if(PYBIND11_FINDPYTHON + AND DEFINED PYTHON_EXECUTABLE + AND NOT DEFINED Python_EXECUTABLE) + set(Python_EXECUTABLE "${PYTHON_EXECUTABLE}") + endif() + + # This is a shortcut that is primarily for the venv cmake preset, + # but can be used to quickly setup tests manually, too + set(PYBIND11_CREATE_WITH_UV + "" + CACHE STRING "Create a virtualenv if it doesn't exist") + + if(NOT PYBIND11_CREATE_WITH_UV STREQUAL "") + set(Python_ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/.venv") + if(EXISTS "${Python_ROOT_DIR}") + if(EXISTS "${CMAKE_BINARY_DIR}/CMakeCache.txt") + message(STATUS "Using existing venv at ${Python_ROOT_DIR}, remove or --fresh to recreate") + else() + # --fresh used to remove the cache + file(REMOVE_RECURSE "${CMAKE_CURRENT_BINARY_DIR}/.venv") + endif() + endif() + if(NOT EXISTS "${Python_ROOT_DIR}") + find_program(UV uv REQUIRED) + # CMake 3.19+ would be able to use COMMAND_ERROR_IS_FATAL + message( + STATUS "Creating venv with ${UV} venv -p ${PYBIND11_CREATE_WITH_UV} '${Python_ROOT_DIR}'") + execute_process(COMMAND ${UV} venv -p ${PYBIND11_CREATE_WITH_UV} "${Python_ROOT_DIR}" + RESULT_VARIABLE _venv_result) + if(_venv_result AND NOT _venv_result EQUAL 0) + message(FATAL_ERROR "uv venv failed with '${_venv_result}'") + endif() + message( + STATUS + "Installing deps with ${UV} pip install -p '${Python_ROOT_DIR}' -r tests/requirements.txt" + ) + execute_process( + COMMAND ${UV} pip install -p "${Python_ROOT_DIR}" -r + "${CMAKE_CURRENT_SOURCE_DIR}/tests/requirements.txt" RESULT_VARIABLE _pip_result) + if(_pip_result AND NOT _pip_result EQUAL 0) + message(FATAL_ERROR "uv pip install failed with '${_pip_result}'") + endif() + endif() + else() + if(NOT DEFINED Python3_EXECUTABLE + AND NOT DEFINED Python_EXECUTABLE + AND NOT DEFINED Python_ROOT_DIR + AND NOT DEFINED ENV{VIRTUALENV} + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.venv") + message(STATUS "Autodetecting Python in virtual environment") + set(Python_ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/.venv") + endif() + endif() + + # Python_ROOT_DIR is only a search hint, so FindPython may select a newer + # Python elsewhere. Set Python_EXECUTABLE to the venv's interpreter path + # to bypass interpreter discovery. + if(DEFINED Python_ROOT_DIR AND NOT DEFINED Python_EXECUTABLE) + if(WIN32 AND EXISTS "${Python_ROOT_DIR}/Scripts/python.exe") + set(Python_EXECUTABLE "${Python_ROOT_DIR}/Scripts/python.exe") + elseif(EXISTS "${Python_ROOT_DIR}/bin/python") + set(Python_EXECUTABLE "${Python_ROOT_DIR}/bin/python") + endif() + endif() +endif() -# NB: when adding a header don't forget to also add it to setup.py set(PYBIND11_HEADERS + include/pybind11/detail/argument_vector.h include/pybind11/detail/class.h include/pybind11/detail/common.h + include/pybind11/detail/cpp_conduit.h include/pybind11/detail/descr.h + include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h + include/pybind11/detail/exception_translation.h + include/pybind11/detail/function_record_pyobject.h + include/pybind11/detail/function_ref.h + include/pybind11/detail/holder_caster_foreign_helpers.h include/pybind11/detail/init.h include/pybind11/detail/internals.h + include/pybind11/detail/native_enum_data.h + include/pybind11/detail/pybind11_namespace_macros.h + include/pybind11/detail/struct_smart_holder.h include/pybind11/detail/type_caster_base.h include/pybind11/detail/typeid.h + include/pybind11/detail/using_smart_holder.h + include/pybind11/detail/value_and_holder.h include/pybind11/attr.h include/pybind11/buffer_info.h include/pybind11/cast.h include/pybind11/chrono.h include/pybind11/common.h include/pybind11/complex.h + include/pybind11/conduit/pybind11_conduit_v1.h + include/pybind11/conduit/pybind11_platform_abi_id.h + include/pybind11/conduit/wrap_include_python_h.h + include/pybind11/critical_section.h include/pybind11/options.h include/pybind11/eigen.h + include/pybind11/eigen/common.h include/pybind11/eigen/matrix.h include/pybind11/eigen/tensor.h include/pybind11/embed.h include/pybind11/eval.h include/pybind11/gil.h + include/pybind11/gil_safe_call_once.h + include/pybind11/gil_simple.h include/pybind11/iostream.h include/pybind11/functional.h + include/pybind11/native_enum.h include/pybind11/numpy.h include/pybind11/operators.h include/pybind11/pybind11.h include/pybind11/pytypes.h + include/pybind11/subinterpreter.h include/pybind11/stl.h include/pybind11/stl_bind.h - include/pybind11/stl/filesystem.h) + include/pybind11/stl/filesystem.h + include/pybind11/trampoline_self_life_support.h + include/pybind11/type_caster_pyobject_ptr.h + include/pybind11/typing.h + include/pybind11/warnings.h) # Compare with grep and warn if mismatched -if(PYBIND11_MASTER_PROJECT AND NOT CMAKE_VERSION VERSION_LESS 3.12) +if(PYBIND11_MASTER_PROJECT) file( GLOB_RECURSE _pybind11_header_check LIST_DIRECTORIES false @@ -160,10 +270,7 @@ if(PYBIND11_MASTER_PROJECT AND NOT CMAKE_VERSION VERSION_LESS 3.12) endif() endif() -# CMake 3.12 added list(TRANSFORM PREPEND -# But we can't use it yet -string(REPLACE "include/" "${CMAKE_CURRENT_SOURCE_DIR}/include/" PYBIND11_HEADERS - "${PYBIND11_HEADERS}") +list(TRANSFORM PYBIND11_HEADERS PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/") # Cache variable so this can be used in parent projects set(pybind11_INCLUDE_DIR @@ -213,11 +320,16 @@ include("${CMAKE_CURRENT_SOURCE_DIR}/tools/JoinPaths.cmake") # Relative directory setting if(USE_PYTHON_INCLUDE_DIR AND DEFINED Python_INCLUDE_DIRS) file(RELATIVE_PATH CMAKE_INSTALL_INCLUDEDIR ${CMAKE_INSTALL_PREFIX} ${Python_INCLUDE_DIRS}) +elseif(USE_PYTHON_INCLUDE_DIR AND DEFINED Python3_INCLUDE_DIRS) + file(RELATIVE_PATH CMAKE_INSTALL_INCLUDEDIR ${CMAKE_INSTALL_PREFIX} ${Python3_INCLUDE_DIRS}) elseif(USE_PYTHON_INCLUDE_DIR AND DEFINED PYTHON_INCLUDE_DIR) - file(RELATIVE_PATH CMAKE_INSTALL_INCLUDEDIR ${CMAKE_INSTALL_PREFIX} ${PYTHON_INCLUDE_DIRS}) + file(RELATIVE_PATH CMAKE_INSTALL_INCLUDEDIR ${CMAKE_INSTALL_PREFIX} ${PYTHON_INCLUDE_DIR}) endif() if(PYBIND11_INSTALL) + if(DEFINED SKBUILD_PROJECT_NAME AND SKBUILD_PROJECT_NAME STREQUAL "pybind11_global") + install(DIRECTORY ${pybind11_INCLUDE_DIR}/pybind11 DESTINATION "${SKBUILD_HEADERS_DIR}") + endif() install(DIRECTORY ${pybind11_INCLUDE_DIR}/pybind11 DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) set(PYBIND11_CMAKECONFIG_INSTALL_DIR "${CMAKE_INSTALL_DATAROOTDIR}/cmake/${PROJECT_NAME}" @@ -233,25 +345,11 @@ if(PYBIND11_INSTALL) tools/${PROJECT_NAME}Config.cmake.in "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake" INSTALL_DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR}) - if(CMAKE_VERSION VERSION_LESS 3.14) - # Remove CMAKE_SIZEOF_VOID_P from ConfigVersion.cmake since the library does - # not depend on architecture specific settings or libraries. - set(_PYBIND11_CMAKE_SIZEOF_VOID_P ${CMAKE_SIZEOF_VOID_P}) - unset(CMAKE_SIZEOF_VOID_P) - - write_basic_package_version_file( - ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake - VERSION ${PROJECT_VERSION} - COMPATIBILITY AnyNewerVersion) - - set(CMAKE_SIZEOF_VOID_P ${_PYBIND11_CMAKE_SIZEOF_VOID_P}) - else() - # CMake 3.14+ natively supports header-only libraries - write_basic_package_version_file( - ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake - VERSION ${PROJECT_VERSION} - COMPATIBILITY AnyNewerVersion ARCH_INDEPENDENT) - endif() + # CMake natively supports header-only libraries + write_basic_package_version_file( + ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake + VERSION ${PROJECT_VERSION} + COMPATIBILITY AnyNewerVersion ARCH_INDEPENDENT) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake @@ -260,6 +358,7 @@ if(PYBIND11_INSTALL) tools/pybind11Common.cmake tools/pybind11Tools.cmake tools/pybind11NewTools.cmake + tools/pybind11GuessPythonExtSuffix.cmake DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR}) if(NOT PYBIND11_EXPORT_NAME) @@ -275,7 +374,21 @@ if(PYBIND11_INSTALL) # pkg-config support if(NOT prefix_for_pc_file) - set(prefix_for_pc_file "${CMAKE_INSTALL_PREFIX}") + if(IS_ABSOLUTE "${CMAKE_INSTALL_DATAROOTDIR}") + set(prefix_for_pc_file "${CMAKE_INSTALL_PREFIX}") + else() + set(pc_datarootdir "${CMAKE_INSTALL_DATAROOTDIR}") + if(CMAKE_VERSION VERSION_LESS 3.20) + set(prefix_for_pc_file "\${pcfiledir}/..") + while(pc_datarootdir) + get_filename_component(pc_datarootdir "${pc_datarootdir}" DIRECTORY) + string(APPEND prefix_for_pc_file "/..") + endwhile() + else() + cmake_path(RELATIVE_PATH CMAKE_INSTALL_PREFIX BASE_DIRECTORY CMAKE_INSTALL_DATAROOTDIR + OUTPUT_VARIABLE prefix_for_pc_file) + endif() + endif() endif() join_paths(includedir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}") configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11.pc.in" @@ -283,6 +396,17 @@ if(PYBIND11_INSTALL) install(FILES "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc" DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig/") + # When building a wheel, include __init__.py's for modules + # (see https://github.com/pybind/pybind11/pull/5552) + if(DEFINED SKBUILD_PROJECT_NAME AND SKBUILD_PROJECT_NAME STREQUAL "pybind11") + file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/empty") + file(TOUCH "${CMAKE_CURRENT_BINARY_DIR}/empty/__init__.py") + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/empty/__init__.py" + DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/") + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/empty/__init__.py" + DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig/") + endif() + # Uninstall target if(PYBIND11_MASTER_PROJECT) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/cmake_uninstall.cmake.in" diff --git a/lib/pybind11/MANIFEST.in b/lib/pybind11/MANIFEST.in deleted file mode 100644 index 033303a..0000000 --- a/lib/pybind11/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -recursive-include pybind11/include/pybind11 *.h -recursive-include pybind11 *.py -recursive-include pybind11 py.typed -include pybind11/share/cmake/pybind11/*.cmake -include LICENSE README.rst pyproject.toml setup.py setup.cfg diff --git a/lib/pybind11/README.rst b/lib/pybind11/README.rst index 3c75edb..c68f03b 100644 --- a/lib/pybind11/README.rst +++ b/lib/pybind11/README.rst @@ -1,9 +1,11 @@ .. figure:: https://github.com/pybind/pybind11/raw/master/docs/pybind11-logo.png :alt: pybind11 logo -**pybind11 — Seamless operability between C++11 and Python** +**pybind11 (v3) — Seamless interoperability between C++ and Python** -|Latest Documentation Status| |Stable Documentation Status| |Gitter chat| |GitHub Discussions| |CI| |Build status| +|Latest Documentation Status| |Stable Documentation Status| |Gitter chat| |GitHub Discussions| + +|CI| |Build status| |SPEC 4 — Using and Creating Nightly Wheels| |Repology| |PyPI package| |Conda-forge| |Python Versions| @@ -34,12 +36,12 @@ dependency. Think of this library as a tiny self-contained version of Boost.Python with everything stripped away that isn't relevant for binding generation. Without comments, the core header files only require ~4K -lines of code and depend on Python (3.6+, or PyPy) and the C++ -standard library. This compact implementation was possible thanks to -some of the new C++11 language features (specifically: tuples, lambda -functions and variadic templates). Since its creation, this library has -grown beyond Boost.Python in many ways, leading to dramatically simpler -binding code in many common situations. +lines of code and depend on Python (CPython 3.9+, PyPy, or GraalPy) and the C++ +standard library. This compact implementation was possible thanks to some C++11 +language features (specifically: tuples, lambda functions and variadic +templates). Since its creation, this library has grown beyond Boost.Python in +many ways, leading to dramatically simpler binding code in many common +situations. Tutorial and reference documentation is provided at `pybind11.readthedocs.io `_. @@ -71,6 +73,7 @@ pybind11 can map the following core C++ features to Python: - Internal references with correct reference counting - C++ classes with virtual (and pure virtual) methods can be extended in Python +- Integrated NumPy support (NumPy 2 requires pybind11 2.12+) Goodies ------- @@ -78,8 +81,9 @@ Goodies In addition to the core functionality, pybind11 provides some extra goodies: -- Python 3.6+, and PyPy3 7.3 are supported with an implementation-agnostic - interface (pybind11 2.9 was the last version to support Python 2 and 3.5). +- CPython 3.9+, PyPy3 7.3.17+, and GraalPy 24.1+ are supported with an + implementation-agnostic interface (see older versions for older CPython + and PyPy versions). - It is possible to bind C++11 lambda functions with captured variables. The lambda capture data is stored inside the resulting @@ -116,28 +120,57 @@ goodies: - With little extra effort, C++ types can be pickled and unpickled similar to regular Python objects. -Supported compilers -------------------- +Supported platforms & compilers +------------------------------- + +pybind11 is exercised in continuous integration across a range of operating +systems, Python versions, C++ standards, and toolchains. For an up-to-date +view of the combinations we currently test, please see the +`pybind11 GitHub Actions `_ +and `AppVeyor `_ logs. -1. Clang/LLVM 3.3 or newer (for Apple Xcode's clang, this is 5.0.0 or - newer) -2. GCC 4.8 or newer -3. Microsoft Visual Studio 2017 or newer -4. Intel classic C++ compiler 18 or newer (ICC 20.2 tested in CI) -5. Cygwin/GCC (previously tested on 2.5.1) -6. NVCC (CUDA 11.0 tested in CI) -7. NVIDIA PGI (20.9 tested in CI) +The test matrix naturally evolves over time as older platforms and compilers +fall out of use and new ones are added by the community. Closely related +versions of a tested compiler or platform will often work as well in practice, +but we cannot promise to validate every possible combination. If a +configuration you rely on is missing from the matrix or regresses, issues and +pull requests to extend coverage are very welcome. At the same time, we need +to balance the size of the test matrix with the available CI resources, +such as GitHub's limits on concurrent jobs under the free tier. About ----- This project was created by `Wenzel Jakob `_. Significant features and/or -improvements to the code were contributed by Jonas Adler, Lori A. Burns, -Sylvain Corlay, Eric Cousineau, Aaron Gokaslan, Ralf Grosse-Kunstleve, Trent Houliston, Axel -Huebl, @hulucc, Yannick Jadoul, Sergey Lyskov Johan Mabille, Tomasz MiÄ…sko, -Dean Moldovan, Ben Pritchard, Jason Rhinelander, Boris Schäling, Pim -Schellart, Henry Schreiner, Ivan Smirnov, Boris Staletic, and Patrick Stewart. +improvements to the code were contributed by +Jonas Adler, +Lori A. Burns, +Sylvain Corlay, +Eric Cousineau, +Aaron Gokaslan, +Ralf Grosse-Kunstleve, +Trent Houliston, +Axel Huebl, +@hulucc, +Yannick Jadoul, +Sergey Lyskov, +Johan Mabille, +Tomasz MiÄ…sko, +Dean Moldovan, +Ben Pritchard, +Jason Rhinelander, +Boris Schäling, +Pim Schellart, +Henry Schreiner, +Ivan Smirnov, +Dustin Spicuzza, +Boris Staletic, +Ethan Steinberg, +Patrick Stewart, +Ivor Wanders, +and +Xiaofei Wang. We thank Google for a generous financial contribution to the continuous integration infrastructure used by this project. @@ -178,3 +211,5 @@ to the terms and conditions of this license. :target: https://pypi.org/project/pybind11/ .. |GitHub Discussions| image:: https://img.shields.io/static/v1?label=Discussions&message=Ask&color=blue&logo=github :target: https://github.com/pybind/pybind11/discussions +.. |SPEC 4 — Using and Creating Nightly Wheels| image:: https://img.shields.io/badge/SPEC-4-green?labelColor=%23004811&color=%235CA038 + :target: https://scientific-python.org/specs/spec-0004/ diff --git a/lib/pybind11/include/pybind11/attr.h b/lib/pybind11/include/pybind11/attr.h index b5e3b7b..d337595 100644 --- a/lib/pybind11/include/pybind11/attr.h +++ b/lib/pybind11/include/pybind11/attr.h @@ -12,6 +12,7 @@ #include "detail/common.h" #include "cast.h" +#include "trampoline_self_life_support.h" #include @@ -26,6 +27,9 @@ struct is_method { explicit is_method(const handle &c) : class_(c) {} }; +/// Annotation for setters +struct is_setter {}; + /// Annotation for operators struct is_operator {}; @@ -78,6 +82,10 @@ struct dynamic_attr {}; /// Annotation which enables the buffer protocol for a type struct buffer_protocol {}; +/// Annotation which enables releasing the GIL before calling the C++ destructor of wrapped +/// instances (pybind/pybind11#1446). +struct release_gil_before_calling_cpp_dtor {}; + /// Annotation which requests that a special metaclass is created for a type struct metaclass { handle value; @@ -185,11 +193,12 @@ struct argument_record { /// Internal data structure which holds metadata about a bound function (signature, overloads, /// etc.) +#define PYBIND11_DETAIL_FUNCTION_RECORD_ABI_ID "v1" // PLEASE UPDATE if the struct is changed. struct function_record { function_record() : is_constructor(false), is_new_style_constructor(false), is_stateless(false), - is_operator(false), is_method(false), has_args(false), has_kwargs(false), - prepend(false) {} + is_operator(false), is_method(false), is_setter(false), has_args(false), + has_kwargs(false), prepend(false) {} /// Function name char *name = nullptr; /* why no C++ strings? They generate heavier code.. */ @@ -230,6 +239,9 @@ struct function_record { /// True if this is a method bool is_method : 1; + /// True if this is a setter + bool is_setter : 1; + /// True if the function has a '*args' argument bool has_args : 1; @@ -261,12 +273,18 @@ struct function_record { /// Pointer to next overload function_record *next = nullptr; }; +// The main purpose of this macro is to make it easy to pin-point the critically related code +// sections. +#define PYBIND11_ENSURE_PRECONDITION_FOR_FUNCTIONAL_H_PERFORMANCE_OPTIMIZATIONS(...) \ + static_assert( \ + __VA_ARGS__, \ + "Violation of precondition for pybind11/functional.h performance optimizations!") /// Special data structure which (temporarily) holds metadata about a bound class struct type_record { PYBIND11_NOINLINE type_record() : multiple_inheritance(false), dynamic_attr(false), buffer_protocol(false), - default_holder(true), module_local(false), is_final(false) {} + module_local(false), is_final(false), release_gil_before_calling_cpp_dtor(false) {} /// Handle to the parent scope handle scope; @@ -295,6 +313,12 @@ struct type_record { /// Function pointer to class_<..>::dealloc void (*dealloc)(detail::value_and_holder &) = nullptr; + /// Function pointer for casting alias class (aka trampoline) pointer to + /// trampoline_self_life_support pointer. Sidesteps cross-DSO RTTI issues + /// on platforms like macOS (see PR #5728 for details). + get_trampoline_self_life_support_fn get_trampoline_self_life_support + = [](void *) -> trampoline_self_life_support * { return nullptr; }; + /// List of base classes of the newly created type list bases; @@ -316,15 +340,17 @@ struct type_record { /// Does the class implement the buffer protocol? bool buffer_protocol : 1; - /// Is the default (unique_ptr) holder type used? - bool default_holder : 1; - /// Is the class definition local to the module shared object? bool module_local : 1; /// Is the class inheritable from python classes? bool is_final : 1; + /// Solves pybind/pybind11#1446 + bool release_gil_before_calling_cpp_dtor : 1; + + holder_enum_t holder_enum_v = holder_enum_t::undefined; + PYBIND11_NOINLINE void add_base(const std::type_info &base, void *(*caster)(void *) ) { auto *base_info = detail::get_type_info(base, false); if (!base_info) { @@ -334,21 +360,25 @@ struct type_record { + "\" referenced unknown base type \"" + tname + "\""); } - if (default_holder != base_info->default_holder) { + // SMART_HOLDER_BAKEIN_FOLLOW_ON: Refine holder compatibility checks. + bool this_has_unique_ptr_holder = (holder_enum_v == holder_enum_t::std_unique_ptr); + bool base_has_unique_ptr_holder + = (base_info->holder_enum_v == holder_enum_t::std_unique_ptr); + if (this_has_unique_ptr_holder != base_has_unique_ptr_holder) { std::string tname(base.name()); detail::clean_type_id(tname); pybind11_fail("generic_type: type \"" + std::string(name) + "\" " - + (default_holder ? "does not have" : "has") + + (this_has_unique_ptr_holder ? "does not have" : "has") + " a non-default holder type while its base \"" + tname + "\" " - + (base_info->default_holder ? "does not" : "does")); + + (base_has_unique_ptr_holder ? "does not" : "does")); } - bases.append((PyObject *) base_info->type); + bases.append(reinterpret_cast(base_info->type)); -#if PY_VERSION_HEX < 0x030B0000 +#ifdef PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET dynamic_attr |= base_info->type->tp_dictoffset != 0; #else - dynamic_attr |= (base_info->type->tp_flags & Py_TPFLAGS_MANAGED_DICT) != 0; + dynamic_attr |= (PyType_GetFlags(base_info->type) & Py_TPFLAGS_MANAGED_DICT) != 0; #endif if (caster) { @@ -426,6 +456,12 @@ struct process_attribute : process_attribute_default { } }; +/// Process an attribute which indicates that this function is a setter +template <> +struct process_attribute : process_attribute_default { + static void init(const is_setter &, function_record *r) { r->is_setter = true; } +}; + /// Process an attribute which indicates the parent scope of a method template <> struct process_attribute : process_attribute_default { @@ -591,6 +627,14 @@ struct process_attribute : process_attribute_default static void init(const module_local &l, type_record *r) { r->module_local = l.value; } }; +template <> +struct process_attribute + : process_attribute_default { + static void init(const release_gil_before_calling_cpp_dtor &, type_record *r) { + r->release_gil_before_calling_cpp_dtor = true; + } +}; + /// Process a 'prepend' attribute, putting this at the beginning of the overload chain template <> struct process_attribute : process_attribute_default { @@ -658,6 +702,12 @@ struct process_attributes { } }; +template +struct is_keep_alive : std::false_type {}; + +template +struct is_keep_alive> : std::true_type {}; + template using is_call_guard = is_instantiation; @@ -671,7 +721,9 @@ template ::value...)> constexpr bool expected_num_args(size_t nargs, bool has_args, bool has_kwargs) { PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(nargs, has_args, has_kwargs); - return named == 0 || (self + named + size_t(has_args) + size_t(has_kwargs)) == nargs; + return named == 0 + || (self + named + static_cast(has_args) + static_cast(has_kwargs)) + == nargs; } PYBIND11_NAMESPACE_END(detail) diff --git a/lib/pybind11/include/pybind11/buffer_info.h b/lib/pybind11/include/pybind11/buffer_info.h index 06120d5..10fa825 100644 --- a/lib/pybind11/include/pybind11/buffer_info.h +++ b/lib/pybind11/include/pybind11/buffer_info.h @@ -37,6 +37,9 @@ inline std::vector f_strides(const std::vector &shape, ssize_t return strides; } +template +struct compare_buffer_info; + PYBIND11_NAMESPACE_END(detail) /// Information record describing a Python buffer object @@ -63,10 +66,11 @@ struct buffer_info { bool readonly = false) : ptr(ptr), itemsize(itemsize), size(1), format(format), ndim(ndim), shape(std::move(shape_in)), strides(std::move(strides_in)), readonly(readonly) { - if (ndim != (ssize_t) shape.size() || ndim != (ssize_t) strides.size()) { + if (ndim != static_cast(shape.size()) + || ndim != static_cast(strides.size())) { pybind11_fail("buffer_info: ndim doesn't match shape and/or strides length"); } - for (size_t i = 0; i < (size_t) ndim; ++i) { + for (size_t i = 0; i < static_cast(ndim); ++i) { size *= shape[i]; } } @@ -99,22 +103,22 @@ struct buffer_info { template buffer_info(const T *ptr, ssize_t size, bool readonly = true) : buffer_info( - const_cast(ptr), sizeof(T), format_descriptor::format(), size, readonly) {} + const_cast(ptr), sizeof(T), format_descriptor::format(), size, readonly) {} explicit buffer_info(Py_buffer *view, bool ownview = true) : buffer_info( - view->buf, - view->itemsize, - view->format, - view->ndim, - {view->shape, view->shape + view->ndim}, - /* Though buffer::request() requests PyBUF_STRIDES, ctypes objects - * ignore this flag and return a view with NULL strides. - * When strides are NULL, build them manually. */ - view->strides - ? std::vector(view->strides, view->strides + view->ndim) - : detail::c_strides({view->shape, view->shape + view->ndim}, view->itemsize), - (view->readonly != 0)) { + view->buf, + view->itemsize, + view->format, + view->ndim, + {view->shape, view->shape + view->ndim}, + /* Though buffer::request() requests PyBUF_STRIDES, ctypes objects + * ignore this flag and return a view with NULL strides. + * When strides are NULL, build them manually. */ + view->strides + ? std::vector(view->strides, view->strides + view->ndim) + : detail::c_strides({view->shape, view->shape + view->ndim}, view->itemsize), + (view->readonly != 0)) { // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) this->m_view = view; // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) @@ -150,6 +154,17 @@ struct buffer_info { Py_buffer *view() const { return m_view; } Py_buffer *&view() { return m_view; } + /* True if the buffer item type is equivalent to `T`. */ + // To define "equivalent" by example: + // `buffer_info::item_type_is_equivalent_to(b)` and + // `buffer_info::item_type_is_equivalent_to(b)` may both be true + // on some platforms, but `int` and `unsigned` will never be equivalent. + // For the ground truth, please inspect `detail::compare_buffer_info<>`. + template + bool item_type_is_equivalent_to() const { + return detail::compare_buffer_info::compare(*this); + } + private: struct private_ctr_tag {}; @@ -162,7 +177,7 @@ struct buffer_info { detail::any_container &&strides_in, bool readonly) : buffer_info( - ptr, itemsize, format, ndim, std::move(shape_in), std::move(strides_in), readonly) {} + ptr, itemsize, format, ndim, std::move(shape_in), std::move(strides_in), readonly) {} Py_buffer *m_view = nullptr; bool ownview = false; @@ -170,9 +185,10 @@ struct buffer_info { PYBIND11_NAMESPACE_BEGIN(detail) -template +template struct compare_buffer_info { static bool compare(const buffer_info &b) { + // NOLINTNEXTLINE(bugprone-sizeof-expression) Needed for `PyObject *` return b.format == format_descriptor::format() && b.itemsize == (ssize_t) sizeof(T); } }; @@ -180,7 +196,7 @@ struct compare_buffer_info { template struct compare_buffer_info::value>> { static bool compare(const buffer_info &b) { - return (size_t) b.itemsize == sizeof(T) + return static_cast(b.itemsize) == sizeof(T) && (b.format == format_descriptor::value || ((sizeof(T) == sizeof(long)) && b.format == (std::is_unsigned::value ? "L" : "l")) diff --git a/lib/pybind11/include/pybind11/cast.h b/lib/pybind11/include/pybind11/cast.h index 3a40460..1d857a0 100644 --- a/lib/pybind11/include/pybind11/cast.h +++ b/lib/pybind11/include/pybind11/cast.h @@ -10,8 +10,11 @@ #pragma once +#include "detail/argument_vector.h" #include "detail/common.h" #include "detail/descr.h" +#include "detail/holder_caster_foreign_helpers.h" +#include "detail/native_enum_data.h" #include "detail/type_caster_base.h" #include "detail/typeid.h" #include "pytypes.h" @@ -42,13 +45,118 @@ using make_caster = type_caster>; // Shortcut for calling a caster's `cast_op_type` cast operator for casting a type_caster to a T template typename make_caster::template cast_op_type cast_op(make_caster &caster) { - return caster.operator typename make_caster::template cast_op_type(); + using result_t = typename make_caster::template cast_op_type; // See PR #4893 + return caster.operator result_t(); } template typename make_caster::template cast_op_type::type> cast_op(make_caster &&caster) { - return std::move(caster).operator typename make_caster:: - template cast_op_type::type>(); + using result_t = typename make_caster::template cast_op_type< + typename std::add_rvalue_reference::type>; // See PR #4893 + return std::move(caster).operator result_t(); +} + +template +class type_caster_enum_type { +private: + using Underlying = typename std::underlying_type::type; + +public: + static constexpr auto name = const_name(); + + template + static handle cast(SrcType &&src, return_value_policy, handle parent) { + handle native_enum + = global_internals_native_enum_type_map_get_item(std::type_index(typeid(EnumType))); + if (native_enum) { + return native_enum(static_cast(src)).release(); + } + return type_caster_base::cast( + std::forward(src), + // Fixes https://github.com/pybind/pybind11/pull/3643#issuecomment-1022987818: + return_value_policy::copy, + parent); + } + + template + static handle cast(SrcType *src, return_value_policy policy, handle parent) { + return cast(*src, policy, parent); + } + + bool load(handle src, bool convert) { + handle native_enum + = global_internals_native_enum_type_map_get_item(std::type_index(typeid(EnumType))); + if (native_enum) { + if (!isinstance(src, native_enum)) { + return false; + } + type_caster underlying_caster; + if (!underlying_caster.load(src.attr("value"), convert)) { + pybind11_fail("native_enum internal consistency failure."); + } + native_value = static_cast(static_cast(underlying_caster)); + native_loaded = true; + return true; + } + + type_caster_base legacy_caster; + if (legacy_caster.load(src, convert)) { + legacy_ptr = static_cast(legacy_caster); + return true; + } + return false; + } + + template + using cast_op_type = detail::cast_op_type; + + // NOLINTNEXTLINE(google-explicit-constructor) + operator EnumType *() { return native_loaded ? &native_value : legacy_ptr; } + + // NOLINTNEXTLINE(google-explicit-constructor) + operator EnumType &() { + if (!native_loaded && !legacy_ptr) { + throw reference_cast_error(); + } + return native_loaded ? native_value : *legacy_ptr; + } + +private: + EnumType native_value; // if loading a py::native_enum + bool native_loaded = false; + EnumType *legacy_ptr = nullptr; // if loading a py::enum_ +}; + +template +struct type_caster_enum_type_enabled : std::true_type {}; + +template +struct type_uses_type_caster_enum_type { + static constexpr bool value + = std::is_enum::value && type_caster_enum_type_enabled::value; +}; + +template +class type_caster::value>> + : public type_caster_enum_type {}; + +template ::value, int> = 0> +bool isinstance_native_enum_impl(handle obj, const std::type_info &tp) { + handle native_enum = global_internals_native_enum_type_map_get_item(tp); + if (!native_enum) { + return false; + } + return isinstance(obj, native_enum); +} + +template ::value, int> = 0> +bool isinstance_native_enum_impl(handle, const std::type_info &) { + return false; +} + +template +bool isinstance_native_enum(handle obj, const std::type_info &tp) { + return isinstance_native_enum_impl>(obj, tp); } template @@ -91,8 +199,7 @@ public: template >::value, \ - int> \ - = 0> \ + int> = 0> \ static ::pybind11::handle cast( \ T_ *src, ::pybind11::return_value_policy policy, ::pybind11::handle parent) { \ if (!src) \ @@ -136,29 +243,28 @@ struct type_caster::value && !is_std_char_t return false; } -#if !defined(PYPY_VERSION) - auto index_check = [](PyObject *o) { return PyIndex_Check(o); }; -#else - // In PyPy 7.3.3, `PyIndex_Check` is implemented by calling `__index__`, - // while CPython only considers the existence of `nb_index`/`__index__`. - auto index_check = [](PyObject *o) { return hasattr(o, "__index__"); }; -#endif - if (std::is_floating_point::value) { - if (convert || PyFloat_Check(src.ptr())) { + if (convert || PyFloat_Check(src.ptr()) || PYBIND11_LONG_CHECK(src.ptr())) { py_value = (py_type) PyFloat_AsDouble(src.ptr()); } else { return false; } } else if (PyFloat_Check(src.ptr()) - || (!convert && !PYBIND11_LONG_CHECK(src.ptr()) && !index_check(src.ptr()))) { + || !(convert || PYBIND11_LONG_CHECK(src.ptr()) + || PYBIND11_INDEX_CHECK(src.ptr()))) { + // Explicitly reject float → int conversion even in convert mode. + // This prevents silent truncation (e.g., 1.9 → 1). + // Only int → float conversion is allowed (widening, no precision loss). + // Also reject if none of the conversion conditions are met. return false; } else { handle src_or_index = src; // PyPy: 7.3.7's 3.8 does not implement PyLong_*'s __index__ calls. -#if PY_VERSION_HEX < 0x03080000 || defined(PYPY_VERSION) +#if defined(PYPY_VERSION) object index; - if (!PYBIND11_LONG_CHECK(src.ptr())) { // So: index_check(src.ptr()) + // If not a PyLong, we need to call PyNumber_Index explicitly on PyPy. + // When convert is false, we only reach here if PYBIND11_INDEX_CHECK passed above. + if (!PYBIND11_LONG_CHECK(src.ptr())) { index = reinterpret_steal(PyNumber_Index(src.ptr())); if (!index) { PyErr_Clear(); @@ -178,8 +284,10 @@ struct type_caster::value && !is_std_char_t } } - // Python API reported an error - bool py_err = py_value == (py_type) -1 && PyErr_Occurred(); + bool py_err = (PyErr_Occurred() != nullptr); + if (py_err) { + assert(py_value == static_cast(-1)); + } // Check to see if the conversion is valid (integers should match exactly) // Signed/unsigned checks happen elsewhere @@ -239,7 +347,12 @@ struct type_caster::value && !is_std_char_t return PyLong_FromUnsignedLongLong((unsigned long long) src); } - PYBIND11_TYPE_CASTER(T, const_name::value>("int", "float")); + PYBIND11_TYPE_CASTER( + T, + io_name::value>("typing.SupportsInt | typing.SupportsIndex", + "int", + "typing.SupportsFloat | typing.SupportsIndex", + "float")); }; template @@ -281,7 +394,8 @@ class type_caster : public type_caster { } /* Check if this is a C++ type */ - const auto &bases = all_type_info((PyTypeObject *) type::handle_of(h).ptr()); + const auto &bases + = all_type_info(reinterpret_cast(type::handle_of(h).ptr())); if (bases.size() == 1) { // Only allowing loading from a single-value type value = values_and_holders(reinterpret_cast(h.ptr())).begin()->value_ptr(); return true; @@ -301,7 +415,7 @@ class type_caster : public type_caster { template using cast_op_type = void *&; explicit operator void *&() { return value; } - static constexpr auto name = const_name("capsule"); + static constexpr auto name = const_name(PYBIND11_CAPSULE_TYPE_TYPE_HINT); private: void *value = nullptr; @@ -325,8 +439,9 @@ class type_caster { value = false; return true; } - if (convert || (std::strcmp("numpy.bool_", Py_TYPE(src.ptr())->tp_name) == 0)) { - // (allow non-implicit conversion for numpy booleans) + if (convert || is_numpy_bool(src)) { + // (allow non-implicit conversion for numpy booleans), use strncmp + // since NumPy 1.x had an additional trailing underscore. Py_ssize_t res = -1; if (src.is_none()) { @@ -340,7 +455,7 @@ class type_caster { #else // Alternate approach for CPython: this does the same as the above, but optimized // using the CPython API so as to avoid an unneeded attribute lookup. - else if (auto *tp_as_number = src.ptr()->ob_type->tp_as_number) { + else if (auto *tp_as_number = Py_TYPE(src.ptr())->tp_as_number) { if (PYBIND11_NB_BOOL(tp_as_number)) { res = (*PYBIND11_NB_BOOL(tp_as_number))(src.ptr()); } @@ -358,6 +473,15 @@ class type_caster { return handle(src ? Py_True : Py_False).inc_ref(); } PYBIND11_TYPE_CASTER(bool, const_name("bool")); + +private: + // Test if an object is a NumPy boolean (without fetching the type). + static bool is_numpy_bool(handle object) { + const char *type_name = Py_TYPE(object.ptr())->tp_name; + // Name changed to `numpy.bool` in NumPy 2, `numpy.bool_` is needed for 1.x support + return std::strcmp("numpy.bool", type_name) == 0 + || std::strcmp("numpy.bool_", type_name) == 0; + } }; // Helper class for UTF-{8,16,32} C++ stl strings: @@ -383,12 +507,11 @@ struct string_caster { static constexpr size_t UTF_N = 8 * sizeof(CharT); bool load(handle src, bool) { - handle load_src = src; if (!src) { return false; } - if (!PyUnicode_Check(load_src.ptr())) { - return load_raw(load_src); + if (!PyUnicode_Check(src.ptr())) { + return load_raw(src); } // For UTF-8 we avoid the need for a temporary `bytes` object by using @@ -396,17 +519,22 @@ struct string_caster { if (UTF_N == 8) { Py_ssize_t size = -1; const auto *buffer - = reinterpret_cast(PyUnicode_AsUTF8AndSize(load_src.ptr(), &size)); + = reinterpret_cast(PyUnicode_AsUTF8AndSize(src.ptr(), &size)); if (!buffer) { PyErr_Clear(); return false; } value = StringType(buffer, static_cast(size)); + if (IsView) { + // `src` owns the buffer; keep it alive if inside a bound function, + // otherwise the caller is responsible for its lifetime. + loader_life_support::try_add_patient(src); + } return true; } auto utfNbytes - = reinterpret_steal(PyUnicode_AsEncodedString(load_src.ptr(), + = reinterpret_steal(PyUnicode_AsEncodedString(src.ptr(), UTF_N == 8 ? "utf-8" : UTF_N == 16 ? "utf-16" : "utf-32", @@ -418,7 +546,7 @@ struct string_caster { const auto *buffer = reinterpret_cast(PYBIND11_BYTES_AS_STRING(utfNbytes.ptr())); - size_t length = (size_t) PYBIND11_BYTES_SIZE(utfNbytes.ptr()) / sizeof(CharT); + size_t length = static_cast(PYBIND11_BYTES_SIZE(utfNbytes.ptr())) / sizeof(CharT); // Skip BOM for UTF-16/32 if (UTF_N > 8) { buffer++; @@ -479,6 +607,9 @@ struct string_caster { pybind11_fail("Unexpected PYBIND11_BYTES_AS_STRING() failure."); } value = StringType(bytes, (size_t) PYBIND11_BYTES_SIZE(src.ptr())); + if (IsView) { + loader_life_support::try_add_patient(src); + } return true; } if (PyByteArray_Check(src.ptr())) { @@ -489,6 +620,9 @@ struct string_caster { pybind11_fail("Unexpected PyByteArray_AsString() failure."); } value = StringType(bytearray, (size_t) PyByteArray_Size(src.ptr())); + if (IsView) { + loader_life_support::try_add_patient(src); + } return true; } @@ -660,8 +794,9 @@ class tuple_caster { return cast(*src, policy, parent); } - static constexpr auto name - = const_name("Tuple[") + concat(make_caster::name...) + const_name("]"); + static constexpr auto name = const_name("tuple[") + + ::pybind11::detail::concat(make_caster::name...) + + const_name("]"); template using cast_op_type = type; @@ -703,7 +838,9 @@ class tuple_caster { cast_impl(T &&src, return_value_policy policy, handle parent, index_sequence) { PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(src, policy, parent); PYBIND11_WORKAROUND_INCORRECT_GCC_UNUSED_BUT_SET_PARAMETER(policy, parent); + std::array entries{{reinterpret_steal( + // NOLINTNEXTLINE(bugprone-use-after-move) make_caster::cast(std::get(std::forward(src)), policy, parent))...}}; for (const auto &entry : entries) { if (!entry) { @@ -727,6 +864,13 @@ class type_caster> : public tuple_caster {} template class type_caster> : public tuple_caster {}; +template <> +class type_caster> : public tuple_caster { +public: + // PEP 484 specifies this syntax for an empty tuple + static constexpr auto name = const_name("tuple[()]"); +}; + /// Helper class which abstracts away certain actions. Users can provide specializations for /// custom holders, but it's only necessary if the type has a non-standard interface. template @@ -734,6 +878,7 @@ struct holder_helper { static auto get(const T &p) -> decltype(p.get()) { return p.get(); } }; +// SMART_HOLDER_BAKEIN_FOLLOW_ON: Rewrite comment, with reference to shared_ptr specialization. /// Type caster for holder types like std::shared_ptr, etc. /// The SFINAE hook is provided to help work around the current lack of support /// for smart-pointer interoperability. Please consider it an implementation @@ -769,16 +914,23 @@ struct copyable_holder_caster : public type_caster_base { protected: friend class type_caster_generic; void check_holder_compat() { - if (typeinfo->default_holder) { + // SMART_HOLDER_BAKEIN_FOLLOW_ON: Refine holder compatibility checks. + bool inst_has_unique_ptr_holder + = (typeinfo->holder_enum_v == holder_enum_t::std_unique_ptr); + if (inst_has_unique_ptr_holder) { throw cast_error("Unable to load a custom holder type from a default-holder instance"); } } - bool load_value(value_and_holder &&v_h) { + bool set_foreign_holder(handle src) { + return holder_caster_foreign_helpers::set_foreign_holder(src, (type *) value, &holder); + } + + void load_value(value_and_holder &&v_h) { if (v_h.holder_constructed()) { value = v_h.value_ptr(); holder = v_h.template holder(); - return true; + return; } throw cast_error("Unable to cast from non-held to held instance (T& to Holder) " #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) @@ -815,10 +967,220 @@ struct copyable_holder_caster : public type_caster_base { holder_type holder; }; +template +struct copyable_holder_caster_shared_ptr_with_smart_holder_support_enabled : std::true_type {}; + +// SMART_HOLDER_BAKEIN_FOLLOW_ON: Refactor copyable_holder_caster to reduce code duplication. +template +struct copyable_holder_caster< + type, + std::shared_ptr, + enable_if_t::value>> + : public type_caster_base { +public: + using base = type_caster_base; + static_assert(std::is_base_of>::value, + "Holder classes are only supported for custom types"); + using base::base; + using base::cast; + using base::typeinfo; + using base::value; + + bool load(handle src, bool convert) { + if (base::template load_impl>>( + src, convert)) { + sh_load_helper.maybe_set_python_instance_is_alias(src); + return true; + } + return false; + } + + explicit operator std::shared_ptr *() { + if (sh_load_helper.was_populated) { + pybind11_fail("Passing `std::shared_ptr *` from Python to C++ is not supported " + "(inherently unsafe)."); + } + return std::addressof(shared_ptr_storage); + } + + explicit operator std::shared_ptr &() { + if (sh_load_helper.was_populated) { + shared_ptr_storage = sh_load_helper.load_as_shared_ptr(typeinfo, value); + } + return shared_ptr_storage; + } + + std::weak_ptr potentially_slicing_weak_ptr() { + if (sh_load_helper.was_populated) { + // Reusing shared_ptr code to minimize code complexity. + shared_ptr_storage + = sh_load_helper.load_as_shared_ptr(typeinfo, + value, + /*responsible_parent=*/nullptr, + /*force_potentially_slicing_shared_ptr=*/true); + } + return shared_ptr_storage; + } + + static handle + cast(const std::shared_ptr &src, return_value_policy policy, handle parent) { + const auto *ptr = src.get(); + typename type_caster_base::cast_sources srcs{ptr}; + if (srcs.creates_smart_holder()) { + return smart_holder_type_caster_support::smart_holder_from_shared_ptr( + src, policy, parent, srcs.result); + } + + auto *tinfo = srcs.result.tinfo; + if (tinfo != nullptr && tinfo->holder_enum_v == holder_enum_t::std_shared_ptr) { + return type_caster_base::cast_holder(srcs, &src); + } + + if (parent) { + return type_caster_generic::cast_non_owning( + srcs, return_value_policy::reference_internal, parent); + } + + throw cast_error("Unable to convert std::shared_ptr to Python when the bound type " + "does not use std::shared_ptr or py::smart_holder as its holder type"); + } + + // This function will succeed even if the `responsible_parent` does not own the + // wrapped C++ object directly. + // It is the responsibility of the caller to ensure that the `responsible_parent` + // has a `keep_alive` relationship with the owner of the wrapped C++ object, or + // that the wrapped C++ object lives for the duration of the process. + static std::shared_ptr shared_ptr_with_responsible_parent(handle responsible_parent) { + copyable_holder_caster loader; + loader.load(responsible_parent, /*convert=*/false); + assert(loader.typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder); + return loader.sh_load_helper.load_as_shared_ptr( + loader.typeinfo, loader.value, responsible_parent); + } + +protected: + friend class type_caster_generic; + void check_holder_compat() { + // SMART_HOLDER_BAKEIN_FOLLOW_ON: Refine holder compatibility checks. + bool inst_has_unique_ptr_holder + = (typeinfo->holder_enum_v == holder_enum_t::std_unique_ptr); + if (inst_has_unique_ptr_holder) { + throw cast_error("Unable to load a custom holder type from a default-holder instance"); + } + } + + bool set_foreign_holder(handle src) { + return holder_caster_foreign_helpers::set_foreign_holder( + src, (type *) value, &shared_ptr_storage); + } + + void load_value(value_and_holder &&v_h) { + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + sh_load_helper.loaded_v_h = v_h; + sh_load_helper.was_populated = true; + value = sh_load_helper.get_void_ptr_or_nullptr(); + return; + } + if (v_h.holder_constructed()) { + value = v_h.value_ptr(); + shared_ptr_storage = v_h.template holder>(); + return; + } + throw cast_error("Unable to cast from non-held to held instance (T& to Holder) " +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) + "(#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for " + "type information)"); +#else + "of type '" + + type_id>() + "''"); +#endif + } + + template , + detail::enable_if_t::value, int> = 0> + bool try_implicit_casts(handle, bool) { + return false; + } + + template , + detail::enable_if_t::value, int> = 0> + bool try_implicit_casts(handle src, bool convert) { + for (auto &cast : typeinfo->implicit_casts) { + copyable_holder_caster sub_caster(*cast.first); + if (sub_caster.load(src, convert)) { + value = cast.second(sub_caster.value); + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + sh_load_helper.loaded_v_h = sub_caster.sh_load_helper.loaded_v_h; + sh_load_helper.was_populated = true; + } else { + shared_ptr_storage + = std::shared_ptr(sub_caster.shared_ptr_storage, (type *) value); + } + return true; + } + } + return false; + } + + static bool try_direct_conversions(handle) { return false; } + + smart_holder_type_caster_support::load_helper> sh_load_helper; // Const2Mutbl + std::shared_ptr shared_ptr_storage; +}; + /// Specialize for the common std::shared_ptr, so users don't need to template class type_caster> : public copyable_holder_caster> {}; +PYBIND11_NAMESPACE_END(detail) + +/// Return a std::shared_ptr with the SAME CONTROL BLOCK as the std::shared_ptr owned by the +/// class_ holder. For class_-wrapped types with trampolines, the returned std::shared_ptr +/// does NOT keep any derived Python objects alive (see issue #1333). +/// +/// For class_-wrapped types using std::shared_ptr as the holder, the following expressions +/// produce equivalent results (see tests/test_potentially_slicing_weak_ptr.cpp,py): +/// +/// - obj.cast>() +/// - py::potentially_slicing_weak_ptr(obj).lock() +/// +/// For class_-wrapped types with trampolines and using py::smart_holder, obj.cast<>() +/// produces a std::shared_ptr that keeps any derived Python objects alive for its own lifetime, +/// but this is achieved by introducing a std::shared_ptr control block that is independent of +/// the one owned by the py::smart_holder. This can lead to surprising std::weak_ptr behavior +/// (see issue #5623). An easy solution is to use py::potentially_slicing_weak_ptr<>(obj), +/// as exercised in tests/test_potentially_slicing_weak_ptr.cpp,py (look for +/// "set_wp_potentially_slicing"). Note, however, that this reintroduces the inheritance +/// slicing issue (see issue #1333). The ideal — but usually more involved — solution is to use +/// a Python weakref to the derived Python object, instead of a C++ base-class std::weak_ptr. +/// +/// It is not possible (at least no known approach exists at the time of this writing) to +/// simultaneously achieve both desirable properties: +/// +/// - the same std::shared_ptr control block as the class_ holder +/// - automatic lifetime extension of any derived Python objects +/// +/// The reason is that this would introduce a reference cycle that cannot be garbage collected: +/// +/// - the derived Python object owns the class_ holder +/// - the class_ holder owns the std::shared_ptr +/// - the std::shared_ptr would own a reference to the derived Python object, +/// completing the cycle +template +std::weak_ptr potentially_slicing_weak_ptr(handle obj) { + detail::make_caster> caster; + if (caster.load(obj, /*convert=*/true)) { + return caster.potentially_slicing_weak_ptr(); + } + const char *obj_type_name = detail::obj_class_name(obj.ptr()); + throw type_error("\"" + std::string(obj_type_name) + + "\" object is not convertible to std::weak_ptr (with T = " + type_id() + + ")"); +} + +PYBIND11_NAMESPACE_BEGIN(detail) + +// SMART_HOLDER_BAKEIN_FOLLOW_ON: Rewrite comment, with reference to unique_ptr specialization. /// Type caster for holder types like std::unique_ptr. /// Please consider the SFINAE hook an implementation detail, as explained /// in the comment for the copyable_holder_caster. @@ -834,6 +1196,141 @@ struct move_only_holder_caster { static constexpr auto name = type_caster_base::name; }; +template +struct move_only_holder_caster_unique_ptr_with_smart_holder_support_enabled : std::true_type {}; + +// SMART_HOLDER_BAKEIN_FOLLOW_ON: Refactor move_only_holder_caster to reduce code duplication. +template +struct move_only_holder_caster< + type, + std::unique_ptr, + enable_if_t::value>> + : public type_caster_base { +public: + using base = type_caster_base; + static_assert(std::is_base_of>::value, + "Holder classes are only supported for custom types"); + using base::base; + using base::cast; + using base::typeinfo; + using base::value; + + static handle + cast(std::unique_ptr &&src, return_value_policy policy, handle parent) { + auto *ptr = src.get(); + typename type_caster_base::cast_sources srcs{ptr}; + if (srcs.creates_smart_holder()) { + return smart_holder_type_caster_support::smart_holder_from_unique_ptr( + std::move(src), policy, parent, srcs.result); + } + return type_caster_base::cast_holder(srcs, &src); + } + + static handle + cast(const std::unique_ptr &src, return_value_policy policy, handle parent) { + if (!src) { + return none().release(); + } + if (policy == return_value_policy::automatic) { + policy = return_value_policy::reference_internal; + } + if (policy != return_value_policy::reference_internal) { + throw cast_error("Invalid return_value_policy for const unique_ptr&"); + } + return type_caster_base::cast(src.get(), policy, parent); + } + + bool load(handle src, bool convert) { + if (base::template load_impl< + move_only_holder_caster>>(src, convert)) { + sh_load_helper.maybe_set_python_instance_is_alias(src); + return true; + } + return false; + } + + bool set_foreign_holder(handle) { + throw cast_error("Foreign instance cannot be converted to std::unique_ptr " + "because we don't know how to make it relinquish " + "ownership"); + } + + void load_value(value_and_holder &&v_h) { + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + sh_load_helper.loaded_v_h = v_h; + sh_load_helper.loaded_v_h.type = typeinfo; + sh_load_helper.was_populated = true; + value = sh_load_helper.get_void_ptr_or_nullptr(); + return; + } + pybind11_fail("Passing `std::unique_ptr` from Python to C++ requires `py::class_` (with T = " + + clean_type_id(typeinfo->cpptype->name()) + ")"); + } + + template + using cast_op_type + = conditional_t::type, + const std::unique_ptr &>::value + || std::is_same::type, + const std::unique_ptr &>::value, + const std::unique_ptr &, + std::unique_ptr>; + + explicit operator std::unique_ptr() { + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + return sh_load_helper.template load_as_unique_ptr(typeinfo, value); + } + pybind11_fail("Expected to be UNREACHABLE: " __FILE__ ":" PYBIND11_TOSTRING(__LINE__)); + } + + explicit operator const std::unique_ptr &() { + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + // Get shared_ptr to ensure that the Python object is not disowned elsewhere. + shared_ptr_storage = sh_load_helper.load_as_shared_ptr(typeinfo, value); + // Build a temporary unique_ptr that is meant to never expire. + unique_ptr_storage = std::shared_ptr>( + new std::unique_ptr{ + sh_load_helper.template load_as_const_unique_ptr( + typeinfo, shared_ptr_storage.get())}, + [](std::unique_ptr *ptr) { + if (!ptr) { + pybind11_fail("FATAL: `const std::unique_ptr &` was disowned " + "(EXPECT UNDEFINED BEHAVIOR)."); + } + (void) ptr->release(); + delete ptr; + }); + return *unique_ptr_storage; + } + pybind11_fail("Expected to be UNREACHABLE: " __FILE__ ":" PYBIND11_TOSTRING(__LINE__)); + } + + bool try_implicit_casts(handle src, bool convert) { + for (auto &cast : typeinfo->implicit_casts) { + move_only_holder_caster sub_caster(*cast.first); + if (sub_caster.load(src, convert)) { + value = cast.second(sub_caster.value); + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + sh_load_helper.loaded_v_h = sub_caster.sh_load_helper.loaded_v_h; + sh_load_helper.was_populated = true; + } else { + pybind11_fail("Expected to be UNREACHABLE: " __FILE__ + ":" PYBIND11_TOSTRING(__LINE__)); + } + return true; + } + } + return false; + } + + static bool try_direct_conversions(handle) { return false; } + + smart_holder_type_caster_support::load_helper> sh_load_helper; // Const2Mutbl + std::shared_ptr shared_ptr_storage; // Serves as a pseudo lock. + std::shared_ptr> unique_ptr_storage; +}; + template class type_caster> : public move_only_holder_caster> {}; @@ -843,18 +1340,20 @@ using type_caster_holder = conditional_t::val copyable_holder_caster, move_only_holder_caster>; -template -struct always_construct_holder { +template +struct always_construct_holder_value { static constexpr bool value = Value; }; +template +struct always_construct_holder : always_construct_holder_value {}; + /// Create a specialization for custom holder types (silently ignores std::shared_ptr) #define PYBIND11_DECLARE_HOLDER_TYPE(type, holder_type, ...) \ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) \ namespace detail { \ template \ - struct always_construct_holder : always_construct_holder { \ - }; \ + struct always_construct_holder : always_construct_holder_value<__VA_ARGS__> {}; \ template \ class type_caster::value>> \ : public type_caster_holder {}; \ @@ -865,14 +1364,61 @@ struct always_construct_holder { template struct is_holder_type : std::is_base_of, detail::type_caster> {}; -// Specialization for always-supported unique_ptr holders: + +// Specializations for always-supported holders: template struct is_holder_type> : std::true_type {}; +template +struct is_holder_type : std::true_type {}; + +#ifdef PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION // See PR #4888 + +// This leads to compilation errors if a specialization is missing. +template +struct handle_type_name; + +#else + template struct handle_type_name { static constexpr auto name = const_name(); }; + +#endif + +template <> +struct handle_type_name { + static constexpr auto name = const_name("object"); +}; +template <> +struct handle_type_name { + static constexpr auto name = const_name("list"); +}; +template <> +struct handle_type_name { + static constexpr auto name = const_name("dict"); +}; +template <> +struct handle_type_name { + static constexpr auto name = const_name("set | frozenset"); +}; +template <> +struct handle_type_name { + static constexpr auto name = const_name("set"); +}; +template <> +struct handle_type_name { + static constexpr auto name = const_name("frozenset"); +}; +template <> +struct handle_type_name { + static constexpr auto name = const_name("str"); +}; +template <> +struct handle_type_name { + static constexpr auto name = const_name("tuple"); +}; template <> struct handle_type_name { static constexpr auto name = const_name("bool"); @@ -882,32 +1428,111 @@ struct handle_type_name { static constexpr auto name = const_name(PYBIND11_BYTES_NAME); }; template <> +struct handle_type_name { + static constexpr auto name = const_name(PYBIND11_BUFFER_TYPE_HINT); +}; +template <> struct handle_type_name { static constexpr auto name = const_name("int"); }; template <> struct handle_type_name { - static constexpr auto name = const_name("Iterable"); + static constexpr auto name = const_name("collections.abc.Iterable"); }; template <> struct handle_type_name { - static constexpr auto name = const_name("Iterator"); + static constexpr auto name = const_name("collections.abc.Iterator"); }; template <> struct handle_type_name { static constexpr auto name = const_name("float"); }; template <> +struct handle_type_name { + static constexpr auto name = const_name("collections.abc.Callable"); +}; +template <> +struct handle_type_name { + static constexpr auto name = handle_type_name::name; +}; +template <> struct handle_type_name { static constexpr auto name = const_name("None"); }; template <> +struct handle_type_name { + static constexpr auto name = const_name("collections.abc.Sequence"); +}; +template <> +struct handle_type_name { + static constexpr auto name = const_name("bytearray"); +}; +template <> +struct handle_type_name { + static constexpr auto name = const_name("memoryview"); +}; +template <> +struct handle_type_name { + static constexpr auto name = const_name("slice"); +}; +template <> +struct handle_type_name { + static constexpr auto name = const_name("type"); +}; +template <> +struct handle_type_name { + static constexpr auto name = const_name(PYBIND11_CAPSULE_TYPE_TYPE_HINT); +}; +template <> +struct handle_type_name { + static constexpr auto name = const_name("ellipsis"); +}; +template <> +struct handle_type_name { + static constexpr auto name = const_name("weakref.ReferenceType"); +}; +// args/Args/kwargs/KWArgs have name as well as typehint included +template <> struct handle_type_name { - static constexpr auto name = const_name("*args"); + static constexpr auto name = io_name("*args", "tuple"); +}; +template +struct handle_type_name> { + static constexpr auto name + = io_name("*args: ", "tuple[") + make_caster::name + io_name("", ", ...]"); }; template <> struct handle_type_name { - static constexpr auto name = const_name("**kwargs"); + static constexpr auto name = io_name("**kwargs", "dict[str, typing.Any]"); +}; +template +struct handle_type_name> { + static constexpr auto name + = io_name("**kwargs: ", "dict[str, ") + make_caster::name + io_name("", "]"); +}; +template <> +struct handle_type_name { + static constexpr auto name = const_name(); +}; +template <> +struct handle_type_name { + static constexpr auto name = const_name(); +}; +template <> +struct handle_type_name { + static constexpr auto name = const_name(); +}; +template <> +struct handle_type_name { + static constexpr auto name = const_name(); +}; +template <> +struct handle_type_name { + static constexpr auto name = const_name(); +}; +template <> +struct handle_type_name { + static constexpr auto name = const_name(); }; template @@ -944,6 +1569,21 @@ struct pyobject_caster { template class type_caster::value>> : public pyobject_caster {}; +template <> +class type_caster : public pyobject_caster { +public: + bool load(handle src, bool /* convert */) { + if (isinstance(src)) { + value = reinterpret_borrow(src); + } else if (isinstance(src)) { + value = float_(reinterpret_borrow(src)); + } else { + return false; + } + return true; + } +}; + // Our conditions for enabling moving are quite restrictive: // At compile time: // - T needs to be a non-const, non-pointer, non-reference type @@ -964,7 +1604,7 @@ struct move_always< enable_if_t< all_of, negation>, - std::is_move_constructible, + is_move_constructible, std::is_same>().operator T &()), T &>>::value>> : std::true_type {}; template @@ -975,7 +1615,7 @@ struct move_if_unreferenced< enable_if_t< all_of, negation>, - std::is_move_constructible, + is_move_constructible, std::is_same>().operator T &()), T &>>::value>> : std::true_type {}; template @@ -1017,11 +1657,14 @@ type_caster &load_type(type_caster &conv, const handle &ha "Internal error: type_caster should only be used for C++ types"); if (!conv.load(handle, true)) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) - throw cast_error("Unable to cast Python instance to C++ type (#define " - "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); + throw cast_error( + "Unable to cast Python instance of type " + + str(type::handle_of(handle)).cast() + + " to C++ type '?' (#define " + "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); #else throw cast_error("Unable to cast Python instance of type " - + (std::string) str(type::handle_of(handle)) + " to C++ type '" + + str(type::handle_of(handle)).cast() + " to C++ type '" + type_id() + "'"); #endif } @@ -1038,11 +1681,23 @@ make_caster load_type(const handle &handle) { PYBIND11_NAMESPACE_END(detail) // pytype -> C++ type -template ::value, int> = 0> +template ::value + && !detail::is_same_ignoring_cvref::value, + int> = 0> T cast(const handle &handle) { using namespace detail; - static_assert(!cast_is_temporary_value_reference::value, + constexpr bool is_enum_cast = type_uses_type_caster_enum_type>::value; + static_assert(!cast_is_temporary_value_reference::value || is_enum_cast, "Unable to cast type to reference: value is local to type caster"); +#ifndef NDEBUG + if (is_enum_cast && cast_is_temporary_value_reference::value) { + if (detail::global_internals_native_enum_type_map_contains( + std::type_index(typeid(intrinsic_t)))) { + pybind11_fail("Unable to cast native enum type to reference"); + } + } +#endif return cast_op(load_type(handle)); } @@ -1052,6 +1707,32 @@ T cast(const handle &handle) { return T(reinterpret_borrow(handle)); } +// Note that `cast(obj)` increments the reference count of `obj`. +// This is necessary for the case that `obj` is a temporary, and could +// not possibly be different, given +// 1. the established convention that the passed `handle` is borrowed, and +// 2. we don't want to force all generic code using `cast()` to special-case +// handling of `T` = `PyObject *` (to increment the reference count there). +// It is the responsibility of the caller to ensure that the reference count +// is decremented. +template ::value + && detail::is_same_ignoring_cvref::value, + int> = 0> +T cast(Handle &&handle) { + return handle.inc_ref().ptr(); +} +// To optimize way an inc_ref/dec_ref cycle: +template ::value + && detail::is_same_ignoring_cvref::value, + int> = 0> +T cast(Object &&obj) { + return obj.release().ptr(); +} + // C++ type -> py::object template ::value, int> = 0> object cast(T &&value, @@ -1085,12 +1766,13 @@ detail::enable_if_t::value, T> move(object &&obj) { if (obj.ref_count() > 1) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) throw cast_error( - "Unable to cast Python instance to C++ rvalue: instance has multiple references" - " (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); + "Unable to cast Python " + str(type::handle_of(obj)).cast() + + " instance to C++ rvalue: instance has multiple references" + " (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); #else - throw cast_error("Unable to move from Python " + (std::string) str(type::handle_of(obj)) - + " instance to C++ " + type_id() - + " instance: instance has multiple references"); + throw cast_error("Unable to move from Python " + + str(type::handle_of(obj)).cast() + " instance to C++ " + + type_id() + " instance: instance has multiple references"); #endif } @@ -1148,12 +1830,42 @@ inline void object::cast() && { PYBIND11_NAMESPACE_BEGIN(detail) +// forward declaration (definition in pybind11.h) +template +std::string generate_type_signature(); + // Declared in pytypes.h: template ::value, int>> object object_or_cast(T &&o) { return pybind11::cast(std::forward(o)); } +// Declared in pytypes.h: +// Implemented here so that make_caster can be used. +template +template +str_attr_accessor object_api::attr_with_type_hint(const char *key) const { +#if !defined(__cpp_inline_variables) + static_assert(always_false::value, + "C++17 feature __cpp_inline_variables not available: " + "https://en.cppreference.com/w/cpp/language/static#Static_data_members"); +#endif + object ann = annotations(); + if (ann.contains(key)) { + throw std::runtime_error("__annotations__[\"" + std::string(key) + "\"] was set already."); + } + + ann[key] = generate_type_signature(); + return {derived(), key}; +} + +template +template +obj_attr_accessor object_api::attr_with_type_hint(handle key) const { + (void) attr_with_type_hint(key.cast().c_str()); + return {derived(), reinterpret_borrow(key)}; +} + // Placeholder type for the unneeded (and dead code) static variable in the // PYBIND11_OVERRIDE_OVERRIDE macro struct override_unused {}; @@ -1179,13 +1891,24 @@ enable_if_t::value, T> cast_ref(object &&, // static_assert, even though if it's in dead code, so we provide a "trampoline" to pybind11::cast // that only does anything in cases where pybind11::cast is valid. template -enable_if_t::value, T> cast_safe(object &&) { +enable_if_t::value + && !detail::is_same_ignoring_cvref::value, + T> +cast_safe(object &&) { pybind11_fail("Internal error: cast_safe fallback invoked"); } template enable_if_t::value, void> cast_safe(object &&) {} template -enable_if_t, std::is_void>::value, T> +enable_if_t::value, PyObject *> +cast_safe(object &&o) { + return o.release().ptr(); +} +template +enable_if_t, + detail::is_same_ignoring_cvref, + std::is_void>::value, + T> cast_safe(object &&o) { return pybind11::cast(std::move(o)); } @@ -1195,9 +1918,10 @@ PYBIND11_NAMESPACE_END(detail) // The overloads could coexist, i.e. the #if is not strictly speaking needed, // but it is an easy minor optimization. #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) -inline cast_error cast_error_unable_to_convert_call_arg() { - return cast_error("Unable to convert call argument to Python object (#define " - "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); +inline cast_error cast_error_unable_to_convert_call_arg(const std::string &name) { + return cast_error("Unable to convert call argument '" + name + + "' to Python object (#define " + "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); } #else inline cast_error cast_error_unable_to_convert_call_arg(const std::string &name, @@ -1207,20 +1931,27 @@ inline cast_error cast_error_unable_to_convert_call_arg(const std::string &name, } #endif +namespace typing { +template +class Tuple : public tuple { + using tuple::tuple; +}; +} // namespace typing + template -tuple make_tuple() { +typing::Tuple<> make_tuple() { return tuple(0); } template -tuple make_tuple(Args &&...args_) { +typing::Tuple make_tuple(Args &&...args_) { constexpr size_t size = sizeof...(Args); std::array args{{reinterpret_steal( detail::make_caster::cast(std::forward(args_), policy, nullptr))...}}; for (size_t i = 0; i < args.size(); i++) { if (!args[i]) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) - throw cast_error_unable_to_convert_call_arg(); + throw cast_error_unable_to_convert_call_arg(std::to_string(i)); #else std::array argtypes{{type_id()...}}; throw cast_error_unable_to_convert_call_arg(std::to_string(i), argtypes[i]); @@ -1232,7 +1963,12 @@ tuple make_tuple(Args &&...args_) { for (auto &arg_value : args) { PyTuple_SET_ITEM(result.ptr(), counter++, arg_value.release().ptr()); } + PYBIND11_WARNING_PUSH +#ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING + PYBIND11_WARNING_DISABLE_CLANG("-Wreturn-std-move") +#endif return result; + PYBIND11_WARNING_POP } /// \ingroup annotations @@ -1324,7 +2060,7 @@ struct kw_only {}; /// \ingroup annotations /// Annotation indicating that all previous arguments are positional-only; the is the equivalent of -/// an unnamed '/' argument (in Python 3.8) +/// an unnamed '/' argument struct pos_only {}; template @@ -1340,7 +2076,15 @@ inline namespace literals { /** \rst String literal version of `arg` \endrst */ -constexpr arg operator"" _a(const char *name, size_t) { return arg(name); } +constexpr arg +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 5 +operator"" _a // gcc 4.8.5 insists on having a space (hard error). +#else +operator""_a // clang 17 generates a deprecation warning if there is a space. +#endif + (const char *name, size_t) { + return arg(name); +} } // namespace literals PYBIND11_NAMESPACE_BEGIN(detail) @@ -1353,6 +2097,9 @@ using is_pos_only = std::is_same, pos_only>; // forward declaration (definition in attr.h) struct function_record; +/// Inline size chosen mostly arbitrarily. +constexpr std::size_t arg_vector_small_size = 6; + /// Internal data associated with a single function call struct function_call { function_call(const function_record &f, handle p); // Implementation in attr.h @@ -1361,10 +2108,10 @@ struct function_call { const function_record &func; /// Arguments passed to the function: - std::vector args; + argument_vector args; /// The `convert` value the arguments should be loaded with - std::vector args_convert; + args_convert_vector args_convert; /// Extra references for the optional `py::args` and/or `py::kwargs` arguments (which, if /// present, are also in `args` but without a reference). @@ -1377,15 +2124,24 @@ struct function_call { handle init_self; }; +// See PR #5396 for the discussion that led to this +template +struct is_same_or_base_of : std::is_same {}; + +// Only evaluate is_base_of if Derived is complete. +// is_base_of raises a compiler error if Derived is incomplete. +template +struct is_same_or_base_of + : any_of, std::is_base_of> {}; + /// Helper class which loads arguments for C++ functions called from Python template class argument_loader { using indices = make_index_sequence; - template - using argument_is_args = std::is_same, args>; + using argument_is_args = is_same_or_base_of>; template - using argument_is_kwargs = std::is_same, kwargs>; + using argument_is_kwargs = is_same_or_base_of>; // Get kwargs argument position, or -1 if not present: static constexpr auto kwargs_pos = constexpr_last(); @@ -1401,7 +2157,8 @@ class argument_loader { static_assert(args_pos == -1 || args_pos == constexpr_first(), "py::args cannot be specified more than once"); - static constexpr auto arg_names = concat(type_descr(make_caster::name)...); + static constexpr auto arg_names + = ::pybind11::detail::concat(type_descr(make_caster::name)...); bool load_args(function_call &call) { return load_impl_sequence(call, indices{}); } @@ -1424,6 +2181,11 @@ class argument_loader { template bool load_impl_sequence(function_call &call, index_sequence) { + PYBIND11_WARNING_PUSH +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 13 + // Work around a GCC -Warray-bounds false positive in argument_vector usage. + PYBIND11_WARNING_DISABLE_GCC("-Warray-bounds") +#endif #ifdef __cpp_fold_expressions if ((... || !std::get(argcasters).load(call.args[Is], call.args_convert[Is]))) { return false; @@ -1435,6 +2197,7 @@ class argument_loader { } } #endif + PYBIND11_WARNING_POP return true; } @@ -1446,86 +2209,121 @@ class argument_loader { std::tuple...> argcasters; }; -/// Helper class which collects only positional arguments for a Python function call. -/// A fancier version below can collect any argument, but this one is optimal for simple calls. +// [workaround(intel)] Separate function required here +// We need to put this into a separate function because the Intel compiler +// fails to compile enable_if_t...>::value> +// (tested with ICC 2021.1 Beta 20200827). +template +constexpr bool args_has_keyword_or_ds() { + return any_of...>::value; +} + +/// Helper class which collects positional, keyword, * and ** arguments for a Python function call template -class simple_collector { +class unpacking_collector { public: template - explicit simple_collector(Ts &&...values) - : m_args(pybind11::make_tuple(std::forward(values)...)) {} - - const tuple &args() const & { return m_args; } - dict kwargs() const { return {}; } + explicit unpacking_collector(Ts &&...values) + : m_names(reinterpret_steal( + handle())) // initialize to null to avoid useless allocation of 0-length tuple + { + /* + Python can sometimes utilize an extra space before the arguments to prepend `self`. + This is important enough that there is a special flag for it: + PY_VECTORCALL_ARGUMENTS_OFFSET. + All we have to do is allocate an extra space at the beginning of this array, and set the + flag. Note that the extra space is not passed directly in to vectorcall. + */ + m_args.reserve(sizeof...(values) + 1); + m_args.push_back_null(); + + if (args_has_keyword_or_ds()) { + list names_list; + + // collect_arguments guarantees this can't be constructed with kwargs before the last + // positional so we don't need to worry about Ts... being in anything but normal python + // order. + using expander = int[]; + (void) expander{0, (process(names_list, std::forward(values)), 0)...}; + + m_names = reinterpret_steal(PyList_AsTuple(names_list.ptr())); + } else { + auto not_used + = reinterpret_steal(handle()); // initialize as null (to avoid an allocation) - tuple args() && { return std::move(m_args); } + using expander = int[]; + (void) expander{0, (process(not_used, std::forward(values)), 0)...}; + } + } /// Call a Python function and pass the collected arguments object call(PyObject *ptr) const { - PyObject *result = PyObject_CallObject(ptr, m_args.ptr()); + size_t nargs = m_args.size() - 1; // -1 for PY_VECTORCALL_ARGUMENTS_OFFSET (see ctor) + if (m_names) { + nargs -= m_names.size(); + } + PyObject *result = PyObject_Vectorcall( + ptr, m_args.data() + 1, nargs | PY_VECTORCALL_ARGUMENTS_OFFSET, m_names.ptr()); if (!result) { throw error_already_set(); } return reinterpret_steal(result); } -private: - tuple m_args; -}; - -/// Helper class which collects positional, keyword, * and ** arguments for a Python function call -template -class unpacking_collector { -public: - template - explicit unpacking_collector(Ts &&...values) { - // Tuples aren't (easily) resizable so a list is needed for collection, - // but the actual function call strictly requires a tuple. - auto args_list = list(); - using expander = int[]; - (void) expander{0, (process(args_list, std::forward(values)), 0)...}; - - m_args = std::move(args_list); + tuple args() const { + size_t nargs = m_args.size() - 1; // -1 for PY_VECTORCALL_ARGUMENTS_OFFSET (see ctor) + if (m_names) { + nargs -= m_names.size(); + } + tuple val(nargs); + for (size_t i = 0; i < nargs; ++i) { + // +1 for PY_VECTORCALL_ARGUMENTS_OFFSET (see ctor) + val[i] = reinterpret_borrow(m_args[i + 1]); + } + return val; } - const tuple &args() const & { return m_args; } - const dict &kwargs() const & { return m_kwargs; } - - tuple args() && { return std::move(m_args); } - dict kwargs() && { return std::move(m_kwargs); } - - /// Call a Python function and pass the collected arguments - object call(PyObject *ptr) const { - PyObject *result = PyObject_Call(ptr, m_args.ptr(), m_kwargs.ptr()); - if (!result) { - throw error_already_set(); + dict kwargs() const { + dict val; + if (m_names) { + size_t offset = m_args.size() - m_names.size(); + for (size_t i = 0; i < m_names.size(); ++i, ++offset) { + val[m_names[i]] = reinterpret_borrow(m_args[offset]); + } } - return reinterpret_steal(result); + return val; } private: + // normal argument, possibly needing conversion template - void process(list &args_list, T &&x) { - auto o = reinterpret_steal( - detail::make_caster::cast(std::forward(x), policy, {})); - if (!o) { + void process(list & /*names_list*/, T &&x) { + handle h = detail::make_caster::cast(std::forward(x), policy, {}); + if (!h) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) - throw cast_error_unable_to_convert_call_arg(); + throw cast_error_unable_to_convert_call_arg(std::to_string(m_args.size() - 1)); #else - throw cast_error_unable_to_convert_call_arg(std::to_string(args_list.size()), + throw cast_error_unable_to_convert_call_arg(std::to_string(m_args.size() - 1), type_id()); #endif } - args_list.append(std::move(o)); + m_args.push_back_steal(h.ptr()); // cast returns a new reference } - void process(list &args_list, detail::args_proxy ap) { + // * unpacking + void process(list & /*names_list*/, detail::args_proxy ap) { + if (!ap) { + return; + } for (auto a : ap) { - args_list.append(a); + m_args.push_back_borrow(a.ptr()); } } - void process(list & /*args_list*/, arg_v a) { + // named argument + // NOLINTNEXTLINE(performance-unnecessary-value-param) + void process(list &names_list, arg_v a) { + assert(names_list); if (!a.name) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) nameless_argument_error(); @@ -1533,7 +2331,8 @@ class unpacking_collector { nameless_argument_error(a.type); #endif } - if (m_kwargs.contains(a.name)) { + auto name = str(a.name); + if (names_list.contains(name)) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) multiple_values_error(); #else @@ -1542,27 +2341,32 @@ class unpacking_collector { } if (!a.value) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) - throw cast_error_unable_to_convert_call_arg(); + throw cast_error_unable_to_convert_call_arg(a.name); #else throw cast_error_unable_to_convert_call_arg(a.name, a.type); #endif } - m_kwargs[a.name] = std::move(a.value); + names_list.append(std::move(name)); + m_args.push_back_borrow(a.value.ptr()); } - void process(list & /*args_list*/, detail::kwargs_proxy kp) { + // ** unpacking + void process(list &names_list, detail::kwargs_proxy kp) { if (!kp) { return; } - for (auto k : reinterpret_borrow(kp)) { - if (m_kwargs.contains(k.first)) { + assert(names_list); + for (auto &&k : reinterpret_borrow(kp)) { + auto name = str(k.first); + if (names_list.contains(name)) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) multiple_values_error(); #else - multiple_values_error(str(k.first)); + multiple_values_error(name); #endif } - m_kwargs[k.first] = k.second; + names_list.append(std::move(name)); + m_args.push_back_borrow(k.second.ptr()); } } @@ -1588,39 +2392,20 @@ class unpacking_collector { } private: - tuple m_args; - dict m_kwargs; + ref_small_vector m_args; + tuple m_names; }; -// [workaround(intel)] Separate function required here -// We need to put this into a separate function because the Intel compiler -// fails to compile enable_if_t...>::value> -// (tested with ICC 2021.1 Beta 20200827). -template -constexpr bool args_are_all_positional() { - return all_of...>::value; -} - -/// Collect only positional arguments for a Python function call -template ()>> -simple_collector collect_arguments(Args &&...args) { - return simple_collector(std::forward(args)...); -} - -/// Collect all arguments, including keywords and unpacking (only instantiated when needed) -template ()>> +/// Collect all arguments, including keywords and unpacking +template unpacking_collector collect_arguments(Args &&...args) { // Following argument order rules for generalized unpacking according to PEP 448 - static_assert(constexpr_last() - < constexpr_first() - && constexpr_last() - < constexpr_first(), - "Invalid function call: positional args must precede keywords and ** unpacking; " - "* unpacking must precede ** unpacking"); + static_assert( + constexpr_last() < constexpr_first(), + "Invalid function call: positional args must precede keywords and */** unpacking;"); + static_assert(constexpr_last() + < constexpr_first(), + "Invalid function call: * unpacking must precede ** unpacking"); return unpacking_collector(std::forward(args)...); } diff --git a/lib/pybind11/include/pybind11/chrono.h b/lib/pybind11/include/pybind11/chrono.h index 167ea0e..668e458 100644 --- a/lib/pybind11/include/pybind11/chrono.h +++ b/lib/pybind11/include/pybind11/chrono.h @@ -63,6 +63,8 @@ class duration_caster { get_duration(const std::chrono::duration &src) { return src; } + static const std::chrono::duration & + get_duration(const std::chrono::duration &&) = delete; // If this is a time_point get the time_since_epoch template @@ -185,7 +187,7 @@ class type_caster> using us_t = duration; auto us = duration_cast(src.time_since_epoch() % seconds(1)); if (us.count() < 0) { - us += seconds(1); + us += duration_cast(seconds(1)); } // Subtract microseconds BEFORE `system_clock::to_time_t`, because: diff --git a/lib/pybind11/include/pybind11/complex.h b/lib/pybind11/include/pybind11/complex.h index 8a831c1..a6ec11a 100644 --- a/lib/pybind11/include/pybind11/complex.h +++ b/lib/pybind11/include/pybind11/complex.h @@ -51,10 +51,28 @@ class type_caster> { if (!src) { return false; } - if (!convert && !PyComplex_Check(src.ptr())) { + if (!convert + && !(PyComplex_Check(src.ptr()) || PyFloat_Check(src.ptr()) + || PYBIND11_LONG_CHECK(src.ptr()))) { return false; } - Py_complex result = PyComplex_AsCComplex(src.ptr()); + handle src_or_index = src; + // PyPy: 7.3.7's 3.8 does not implement PyLong_*'s __index__ calls. + // The same logic is used in numeric_caster for ints and floats +#if defined(PYPY_VERSION) + object index; + if (PYBIND11_INDEX_CHECK(src.ptr())) { + index = reinterpret_steal(PyNumber_Index(src.ptr())); + if (!index) { + PyErr_Clear(); + if (!convert) + return false; + } else { + src_or_index = index; + } + } +#endif + Py_complex result = PyComplex_AsCComplex(src_or_index.ptr()); if (result.real == -1.0 && PyErr_Occurred()) { PyErr_Clear(); return false; @@ -68,7 +86,12 @@ class type_caster> { return PyComplex_FromDoubles((double) src.real(), (double) src.imag()); } - PYBIND11_TYPE_CASTER(std::complex, const_name("complex")); + // `complex` does not satisfy `typing.SupportsComplex` in typeshed for Python <= 3.10. + // Keep it explicit so generated stubs targeting those versions accept complex values. + PYBIND11_TYPE_CASTER( + std::complex, + io_name("complex | typing.SupportsComplex | typing.SupportsFloat | typing.SupportsIndex", + "complex")); }; PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/conduit/README.txt b/lib/pybind11/include/pybind11/conduit/README.txt new file mode 100644 index 0000000..9a2c53b --- /dev/null +++ b/lib/pybind11/include/pybind11/conduit/README.txt @@ -0,0 +1,15 @@ +NOTE +---- + +The C++ code here + +** only depends on ** + +and nothing else. + +DO NOT ADD CODE WITH OTHER EXTERNAL DEPENDENCIES TO THIS DIRECTORY. + +Read on: + +pybind11_conduit_v1.h — Type-safe interoperability between different + independent Python/C++ bindings systems. diff --git a/lib/pybind11/include/pybind11/conduit/pybind11_conduit_v1.h b/lib/pybind11/include/pybind11/conduit/pybind11_conduit_v1.h new file mode 100644 index 0000000..6d5d0ef --- /dev/null +++ b/lib/pybind11/include/pybind11/conduit/pybind11_conduit_v1.h @@ -0,0 +1,116 @@ +// Copyright (c) 2024 The pybind Community. + +/* The pybind11_conduit_v1 feature enables type-safe interoperability between + +* different independent Python/C++ bindings systems, + +* including pybind11 versions with different PYBIND11_INTERNALS_VERSION's. + + * NOTE: The conduit feature + only covers from-Python-to-C++ conversions, it + does not cover from-C++-to-Python conversions. + (For the latter, a different feature would have to be added.) + +The naming of the feature is a bit misleading: + +* The feature is in no way tied to pybind11 internals. + +* It just happens to originate from pybind11 and currently still lives there. + +* The only external dependency is . + +The implementation is a VERY light-weight dependency. It is designed to be +compatible with any ISO C++11 (or higher) compiler, and does NOT require +C++ Exception Handling to be enabled. + +Please see https://github.com/pybind/pybind11/pull/5296 for more background. + +The implementation involves a + +def _pybind11_conduit_v1_( + self, + pybind11_platform_abi_id: bytes, + cpp_type_info_capsule: capsule, + pointer_kind: bytes) -> capsule + +method that is meant to be added to Python objects wrapping C++ objects +(e.g. pybind11::class_-wrapped types). + +The design of the _pybind11_conduit_v1_ feature provides two layers of +protection against C++ ABI mismatches: + +* The first and most important layer is that the pybind11_platform_abi_id's + must match between extensions. — This will never be perfect, but is the same + pragmatic approach used in pybind11 since 2017 + (https://github.com/pybind/pybind11/commit/96997a4b9d4ec3d389a570604394af5d5eee2557, + PYBIND11_INTERNALS_ID). + +* The second layer is that the typeid(std::type_info).name()'s must match + between extensions. + +The implementation below (which is shorter than this comment!), serves as a +battle-tested specification. The main API is this one function: + +auto *cpp_pointer = pybind11_conduit_v1::get_type_pointer_ephemeral(py_obj); + +It is meant to be a minimalistic reference implementation, intentionally +without comprehensive error reporting. It is expected that major bindings +systems will roll their own, compatible implementations, potentially with +system-specific error reporting. The essential specifications all bindings +systems need to agree on are merely: + +* PYBIND11_PLATFORM_ABI_ID (const char* literal). + +* The cpp_type_info capsule (see below: a void *ptr and a const char *name). + +* The cpp_conduit capsule (see below: a void *ptr and a const char *name). + +* "raw_pointer_ephemeral" means: the lifetime of the pointer is the lifetime + of the py_obj. + +*/ + +// THIS MUST STAY AT THE TOP! +#include "pybind11_platform_abi_id.h" + +#include +#include + +namespace pybind11_conduit_v1 { + +inline void *get_raw_pointer_ephemeral(PyObject *py_obj, const std::type_info *cpp_type_info) { + PyObject *cpp_type_info_capsule + = PyCapsule_New(const_cast(static_cast(cpp_type_info)), + typeid(std::type_info).name(), + nullptr); + if (cpp_type_info_capsule == nullptr) { + return nullptr; + } + PyObject *cpp_conduit = PyObject_CallMethod(py_obj, + "_pybind11_conduit_v1_", + "yOy", + PYBIND11_PLATFORM_ABI_ID, + cpp_type_info_capsule, + "raw_pointer_ephemeral"); + Py_DECREF(cpp_type_info_capsule); + if (cpp_conduit == nullptr) { + return nullptr; + } + void *raw_ptr = PyCapsule_GetPointer(cpp_conduit, cpp_type_info->name()); + Py_DECREF(cpp_conduit); + if (PyErr_Occurred()) { + return nullptr; + } + return raw_ptr; +} + +template +T *get_type_pointer_ephemeral(PyObject *py_obj) { + void *raw_ptr = get_raw_pointer_ephemeral(py_obj, &typeid(T)); + if (raw_ptr == nullptr) { + return nullptr; + } + return static_cast(raw_ptr); +} + +} // namespace pybind11_conduit_v1 diff --git a/lib/pybind11/include/pybind11/conduit/pybind11_platform_abi_id.h b/lib/pybind11/include/pybind11/conduit/pybind11_platform_abi_id.h new file mode 100644 index 0000000..d21fdc5 --- /dev/null +++ b/lib/pybind11/include/pybind11/conduit/pybind11_platform_abi_id.h @@ -0,0 +1,87 @@ +#pragma once + +// Copyright (c) 2024 The pybind Community. + +// To maximize reusability: +// DO NOT ADD CODE THAT REQUIRES C++ EXCEPTION HANDLING. + +#include "wrap_include_python_h.h" + +// Implementation details. DO NOT USE ELSEWHERE. (Unfortunately we cannot #undef them.) +// This is duplicated here to maximize portability. +#define PYBIND11_PLATFORM_ABI_ID_STRINGIFY(x) #x +#define PYBIND11_PLATFORM_ABI_ID_TOSTRING(x) PYBIND11_PLATFORM_ABI_ID_STRINGIFY(x) + +#ifdef PYBIND11_COMPILER_TYPE +// // To maintain backward compatibility (see PR #5439). +# define PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE "" +#else +# define PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE "_" +# if defined(__MINGW32__) +# define PYBIND11_COMPILER_TYPE "mingw" +# elif defined(__CYGWIN__) +# define PYBIND11_COMPILER_TYPE "gcc_cygwin" +# elif defined(_MSC_VER) +# define PYBIND11_COMPILER_TYPE "msvc" +# elif defined(__clang__) || defined(__GNUC__) +# define PYBIND11_COMPILER_TYPE "system" // Assumed compatible with system compiler. +# else +# error "Unknown PYBIND11_COMPILER_TYPE: PLEASE REVISE THIS CODE." +# endif +#endif + +// PR #5439 made this macro obsolete. However, there are many manipulations of this macro in the +// wild. Therefore, to maintain backward compatibility, it is kept around. +#ifndef PYBIND11_STDLIB +# define PYBIND11_STDLIB "" +#endif + +#ifndef PYBIND11_BUILD_ABI +# if defined(_MSC_VER) // See PR #4953. +# if defined(_MT) && defined(_DLL) // Corresponding to CL command line options /MD or /MDd. +# if (_MSC_VER) / 100 == 19 +# define PYBIND11_BUILD_ABI "_md_mscver19" +# else +# error "Unknown major version for MSC_VER: PLEASE REVISE THIS CODE." +# endif +# elif defined(_MT) // Corresponding to CL command line options /MT or /MTd. +# define PYBIND11_BUILD_ABI "_mt_mscver" PYBIND11_PLATFORM_ABI_ID_TOSTRING(_MSC_VER) +# else +# if (_MSC_VER) / 100 == 19 +# define PYBIND11_BUILD_ABI "_none_mscver19" +# else +# error "Unknown major version for MSC_VER: PLEASE REVISE THIS CODE." +# endif +# endif +# elif defined(_LIBCPP_ABI_VERSION) // https://libcxx.llvm.org/DesignDocs/ABIVersioning.html +# define PYBIND11_BUILD_ABI \ + "_libcpp_abi" PYBIND11_PLATFORM_ABI_ID_TOSTRING(_LIBCPP_ABI_VERSION) +# elif defined(_GLIBCXX_USE_CXX11_ABI) // See PR #5439. +# if defined(__NVCOMPILER) +// // Assume that NVHPC is in the 1xxx ABI family. +// // THIS ASSUMPTION IS NOT FUTURE PROOF but apparently the best we can do. +// // Please let us know if there is a way to validate the assumption here. +# elif !defined(__GXX_ABI_VERSION) +# error \ + "Unknown platform or compiler (_GLIBCXX_USE_CXX11_ABI): PLEASE REVISE THIS CODE." +# endif +# if defined(__GXX_ABI_VERSION) && __GXX_ABI_VERSION < 1002 || __GXX_ABI_VERSION >= 2000 +# error "Unknown platform or compiler (__GXX_ABI_VERSION): PLEASE REVISE THIS CODE." +# endif +# define PYBIND11_BUILD_ABI \ + "_libstdcpp_gxx_abi_1xxx_use_cxx11_abi_" PYBIND11_PLATFORM_ABI_ID_TOSTRING( \ + _GLIBCXX_USE_CXX11_ABI) +# else +# error "Unknown platform or compiler: PLEASE REVISE THIS CODE." +# endif +#endif + +// On MSVC, debug and release builds are not ABI-compatible! +#if defined(_MSC_VER) && defined(_DEBUG) +# define PYBIND11_BUILD_TYPE "_debug" +#else +# define PYBIND11_BUILD_TYPE "" +#endif + +#define PYBIND11_PLATFORM_ABI_ID \ + PYBIND11_COMPILER_TYPE PYBIND11_STDLIB PYBIND11_BUILD_ABI PYBIND11_BUILD_TYPE diff --git a/lib/pybind11/include/pybind11/conduit/wrap_include_python_h.h b/lib/pybind11/include/pybind11/conduit/wrap_include_python_h.h new file mode 100644 index 0000000..713dc4b --- /dev/null +++ b/lib/pybind11/include/pybind11/conduit/wrap_include_python_h.h @@ -0,0 +1,72 @@ +#pragma once + +// Copyright (c) 2024 The pybind Community. + +// STRONG REQUIREMENT: +// This header is a wrapper around `#include `, therefore it +// MUST BE INCLUDED BEFORE ANY STANDARD HEADERS are included. +// See also: +// https://docs.python.org/3/c-api/intro.html#include-files +// Quoting from there: +// Note: Since Python may define some pre-processor definitions which affect +// the standard headers on some systems, you must include Python.h before +// any standard headers are included. + +// To maximize reusability: +// DO NOT ADD CODE THAT REQUIRES C++ EXCEPTION HANDLING. + +// Disable linking to pythonX_d.lib on Windows in debug mode. +#if defined(_MSC_VER) && defined(_DEBUG) && !defined(Py_DEBUG) +// Workaround for a VS 2022 issue. +// See https://github.com/pybind/pybind11/pull/3497 for full context. +// NOTE: This workaround knowingly violates the Python.h include order +// requirement (see above). +# include +# if _MSVC_STL_VERSION >= 143 +# include +# endif +# define PYBIND11_DEBUG_MARKER +# undef _DEBUG +#endif + +// Don't let Python.h #define (v)snprintf as macro because they are implemented +// properly in Visual Studio since 2015. +#if defined(_MSC_VER) +# define HAVE_SNPRINTF 1 +#endif + +#if defined(_MSC_VER) +# pragma warning(push) +# pragma warning(disable : 4505) +// C4505: 'PySlice_GetIndicesEx': unreferenced local function has been removed +#endif + +#include +#include +#include + +#if defined(_MSC_VER) +# pragma warning(pop) +#endif + +#if defined(PYBIND11_DEBUG_MARKER) +# define _DEBUG 1 +# undef PYBIND11_DEBUG_MARKER +#endif + +// Python #defines overrides on all sorts of core functions, which +// tends to wreak havok in C++ codebases that expect these to work +// like regular functions (potentially with several overloads). +#if defined(isalnum) +# undef isalnum +# undef isalpha +# undef islower +# undef isspace +# undef isupper +# undef tolower +# undef toupper +#endif + +#if defined(copysign) +# undef copysign +#endif diff --git a/lib/pybind11/include/pybind11/critical_section.h b/lib/pybind11/include/pybind11/critical_section.h new file mode 100644 index 0000000..2d26412 --- /dev/null +++ b/lib/pybind11/include/pybind11/critical_section.h @@ -0,0 +1,56 @@ +// Copyright (c) 2016-2025 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#pragma once + +#include "pytypes.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +/// This does not do anything if there's a GIL. On free-threaded Python, +/// it locks an object. This uses the CPython API, which has limits +class scoped_critical_section { +public: +#ifdef Py_GIL_DISABLED + explicit scoped_critical_section(handle obj1, handle obj2 = handle{}) { + if (obj1) { + if (obj2) { + PyCriticalSection2_Begin(§ion2, obj1.ptr(), obj2.ptr()); + rank = 2; + } else { + PyCriticalSection_Begin(§ion, obj1.ptr()); + rank = 1; + } + } else if (obj2) { + PyCriticalSection_Begin(§ion, obj2.ptr()); + rank = 1; + } + } + + ~scoped_critical_section() { + if (rank == 1) { + PyCriticalSection_End(§ion); + } else if (rank == 2) { + PyCriticalSection2_End(§ion2); + } + } +#else + explicit scoped_critical_section(handle, handle = handle{}) {}; + ~scoped_critical_section() = default; +#endif + + scoped_critical_section(const scoped_critical_section &) = delete; + scoped_critical_section &operator=(const scoped_critical_section &) = delete; + +private: +#ifdef Py_GIL_DISABLED + int rank{0}; + union { + PyCriticalSection section; + PyCriticalSection2 section2; + }; +#endif +}; + +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/detail/argument_vector.h b/lib/pybind11/include/pybind11/detail/argument_vector.h new file mode 100644 index 0000000..6e2c2ec --- /dev/null +++ b/lib/pybind11/include/pybind11/detail/argument_vector.h @@ -0,0 +1,417 @@ +/* + pybind11/detail/argument_vector.h: small_vector-like containers to + avoid heap allocation of arguments during function call dispatch. + + Copyright (c) Meta Platforms, Inc. and affiliates. + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include + +#include "common.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_WARNING_DISABLE_MSVC(4127) + +PYBIND11_NAMESPACE_BEGIN(detail) + +// Shared implementation utility for our small_vector-like containers. +// We support C++11 and C++14, so we cannot use +// std::variant. Union with the tag packed next to the inline +// array's size is smaller anyway, allowing 1 extra handle of +// inline storage for free. Compare the layouts (1 line per +// size_t/void*, assuming a 64-bit machine): +// With variant, total is N + 2 for N >= 2: +// - variant tag (cannot be packed with the array size) +// - array size (or first pointer of 3 in std::vector) +// - N pointers of inline storage (or 2 remaining pointers of std::vector) +// Custom union, total is N + 1 for N >= 3: +// - variant tag & array size if applicable +// - N pointers of inline storage (or 3 pointers of std::vector) +// +// NOTE: this is a low-level representational convenience; the two +// use cases of this union are materially different and in particular +// have different semantics for inline_array::size. All that is being +// shared is the memory management behavior. +template +union inline_array_or_vector { + struct inline_array { + bool is_inline = true; + std::uint32_t size = 0; + std::array arr; + }; + struct heap_vector { + bool is_inline = false; + std::vector vec; + + heap_vector() = default; + heap_vector(std::size_t count, VectorT value) : vec(count, value) {} + }; + + inline_array iarray; + heap_vector hvector; + + inline_array_or_vector() : iarray() {} + + ~inline_array_or_vector() { + if (is_inline()) { + iarray.~inline_array(); + } else { + hvector.~heap_vector(); + } + } + + // Disable copy ctor and assignment. + inline_array_or_vector(const inline_array_or_vector &) = delete; + inline_array_or_vector &operator=(const inline_array_or_vector &) = delete; + + inline_array_or_vector(inline_array_or_vector &&rhs) noexcept { + if (rhs.is_inline()) { + new (&iarray) inline_array(std::move(rhs.iarray)); + } else { + new (&hvector) heap_vector(std::move(rhs.hvector)); + } + assert(is_inline() == rhs.is_inline()); + } + + inline_array_or_vector &operator=(inline_array_or_vector &&rhs) noexcept { + if (this == &rhs) { + return *this; + } + + if (is_inline()) { + iarray.~inline_array(); + } else { + hvector.~heap_vector(); + } + + if (rhs.is_inline()) { + new (&iarray) inline_array(std::move(rhs.iarray)); + } else { + new (&hvector) heap_vector(std::move(rhs.hvector)); + } + return *this; + } + + bool is_inline() const { + // It is undefined behavior to access the inactive member of a + // union directly. However, it is well-defined to reinterpret_cast any + // pointer into a pointer to char and examine it as an array + // of bytes. See + // https://dev-discuss.pytorch.org/t/unionizing-for-profit-how-to-exploit-the-power-of-unions-in-c/444#the-memcpy-loophole-4 + bool result = false; + static_assert(offsetof(inline_array, is_inline) == 0, + "untagged union implementation relies on this"); + static_assert(offsetof(heap_vector, is_inline) == 0, + "untagged union implementation relies on this"); + std::memcpy(&result, reinterpret_cast(this), sizeof(bool)); + return result; + } +}; + +template +struct small_vector { +public: + small_vector() = default; + + // Disable copy ctor and assignment. + small_vector(const small_vector &) = delete; + small_vector &operator=(const small_vector &) = delete; + small_vector(small_vector &&) noexcept = default; + small_vector &operator=(small_vector &&) noexcept = default; + + std::size_t size() const { + if (is_inline()) { + return m_repr.iarray.size; + } + return m_repr.hvector.vec.size(); + } + + T const *data() const { + if (is_inline()) { + return m_repr.iarray.arr.data(); + } + return m_repr.hvector.vec.data(); + } + + T &operator[](std::size_t idx) { + assert(idx < size()); + if (is_inline()) { + return m_repr.iarray.arr[idx]; + } + return m_repr.hvector.vec[idx]; + } + + T const &operator[](std::size_t idx) const { + assert(idx < size()); + if (is_inline()) { + return m_repr.iarray.arr[idx]; + } + return m_repr.hvector.vec[idx]; + } + + void push_back(const T &x) { emplace_back(x); } + + void push_back(T &&x) { emplace_back(std::move(x)); } + + template + void emplace_back(Args &&...x) { + if (is_inline()) { + auto &ha = m_repr.iarray; + if (ha.size == InlineSize) { + move_to_heap_vector_with_reserved_size(InlineSize + 1); + m_repr.hvector.vec.emplace_back(std::forward(x)...); + } else { + ha.arr[ha.size++] = T(std::forward(x)...); + } + } else { + m_repr.hvector.vec.emplace_back(std::forward(x)...); + } + } + + void reserve(std::size_t sz) { + if (is_inline()) { + if (sz > InlineSize) { + move_to_heap_vector_with_reserved_size(sz); + } + } else { + reserve_slow_path(sz); + } + } + +private: + using repr_type = inline_array_or_vector; + repr_type m_repr; + + PYBIND11_NOINLINE void move_to_heap_vector_with_reserved_size(std::size_t reserved_size) { + assert(is_inline()); + auto &ha = m_repr.iarray; + using heap_vector = typename repr_type::heap_vector; + heap_vector hv; + hv.vec.reserve(reserved_size); + static_assert(std::is_nothrow_move_constructible::value, + "this conversion is not exception safe"); + static_assert(std::is_nothrow_move_constructible::value, + "this conversion is not exception safe"); + std::move(ha.arr.begin(), ha.arr.begin() + ha.size, std::back_inserter(hv.vec)); + new (&m_repr.hvector) heap_vector(std::move(hv)); + } + + PYBIND11_NOINLINE void reserve_slow_path(std::size_t sz) { m_repr.hvector.vec.reserve(sz); } + + bool is_inline() const { return m_repr.is_inline(); } +}; + +// Container to avoid heap allocation for kRequestedInlineSize or fewer booleans. +template +struct small_vector { +private: +public: + small_vector() = default; + + // Disable copy ctor and assignment. + small_vector(const small_vector &) = delete; + small_vector &operator=(const small_vector &) = delete; + small_vector(small_vector &&) noexcept = default; + small_vector &operator=(small_vector &&) noexcept = default; + + small_vector(std::size_t count, bool value) { + if (count > kInlineSize) { + new (&m_repr.hvector) typename repr_type::heap_vector(count, value); + } else { + auto &inline_arr = m_repr.iarray; + inline_arr.arr.fill(value ? static_cast(-1) : 0); + inline_arr.size = static_cast(count); + } + } + + std::size_t size() const { + if (is_inline()) { + return m_repr.iarray.size; + } + return m_repr.hvector.vec.size(); + } + + void reserve(std::size_t sz) { + if (is_inline()) { + if (sz > kInlineSize) { + move_to_heap_vector_with_reserved_size(sz); + } + } else { + m_repr.hvector.vec.reserve(sz); + } + } + + bool operator[](std::size_t idx) const { + if (is_inline()) { + return inline_index(idx); + } + assert(idx < m_repr.hvector.vec.size()); + return m_repr.hvector.vec[idx]; + } + + void push_back(bool b) { + if (is_inline()) { + auto &ha = m_repr.iarray; + if (ha.size == kInlineSize) { + move_to_heap_vector_with_reserved_size(kInlineSize + 1); + push_back_slow_path(b); + } else { + assert(ha.size < kInlineSize); + const auto wbi = word_and_bit_index(ha.size++); + assert(wbi.word < kWords); + assert(wbi.bit < kBitsPerWord); + if (b) { + ha.arr[wbi.word] |= (static_cast(1) << wbi.bit); + } else { + ha.arr[wbi.word] &= ~(static_cast(1) << wbi.bit); + } + assert(operator[](ha.size - 1) == b); + } + } else { + push_back_slow_path(b); + } + } + + void set(std::size_t idx, bool value = true) { + if (is_inline()) { + auto &ha = m_repr.iarray; + assert(ha.size < kInlineSize); + const auto wbi = word_and_bit_index(idx); + assert(wbi.word < kWords); + assert(wbi.bit < kBitsPerWord); + if (value) { + ha.arr[wbi.word] |= (static_cast(1) << wbi.bit); + } else { + ha.arr[wbi.word] &= ~(static_cast(1) << wbi.bit); + } + } else { + m_repr.hvector.vec[idx] = value; + } + } + + void swap(small_vector &rhs) noexcept { std::swap(m_repr, rhs.m_repr); } + +private: + struct WordAndBitIndex { + std::size_t word; + std::size_t bit; + }; + + static WordAndBitIndex word_and_bit_index(std::size_t idx) { + return WordAndBitIndex{idx / kBitsPerWord, idx % kBitsPerWord}; + } + + bool inline_index(std::size_t idx) const { + const auto wbi = word_and_bit_index(idx); + assert(wbi.word < kWords); + assert(wbi.bit < kBitsPerWord); + return m_repr.iarray.arr[wbi.word] & (static_cast(1) << wbi.bit); + } + + PYBIND11_NOINLINE void move_to_heap_vector_with_reserved_size(std::size_t reserved_size) { + auto &inline_arr = m_repr.iarray; + using heap_vector = typename repr_type::heap_vector; + heap_vector hv; + hv.vec.reserve(reserved_size); + for (std::size_t ii = 0; ii < inline_arr.size; ++ii) { + hv.vec.push_back(inline_index(ii)); + } + new (&m_repr.hvector) heap_vector(std::move(hv)); + } + + PYBIND11_NOINLINE void push_back_slow_path(bool b) { m_repr.hvector.vec.push_back(b); } + + static constexpr auto kBitsPerWord = 8 * sizeof(std::size_t); + static constexpr auto kWords = (kRequestedInlineSize + kBitsPerWord - 1) / kBitsPerWord; + static constexpr auto kInlineSize = kWords * kBitsPerWord; + + using repr_type = inline_array_or_vector; + repr_type m_repr; + + bool is_inline() const { return m_repr.is_inline(); } +}; + +// Container to avoid heap allocation for N or fewer arguments. +template +using argument_vector = small_vector; + +// Container to avoid heap allocation for N or fewer booleans. +template +using args_convert_vector = small_vector; + +/// A small_vector of PyObject* that holds references and releases them on destruction. +/// This provides explicit ownership semantics without relying on py::object's +/// destructor, and avoids the need for reinterpret_cast when passing to vectorcall. +template +class ref_small_vector { +public: + ref_small_vector() = default; + + ~ref_small_vector() { + for (std::size_t i = 0; i < m_ptrs.size(); ++i) { + Py_XDECREF(m_ptrs[i]); + } + } + + // Disable copy (prevent accidental double-decref) + ref_small_vector(const ref_small_vector &) = delete; + ref_small_vector &operator=(const ref_small_vector &) = delete; + + // Move is allowed + ref_small_vector(ref_small_vector &&other) noexcept : m_ptrs(std::move(other.m_ptrs)) { + // other.m_ptrs is now empty, so its destructor won't decref anything + } + + ref_small_vector &operator=(ref_small_vector &&other) noexcept { + if (this != &other) { + // Decref our current contents + for (std::size_t i = 0; i < m_ptrs.size(); ++i) { + Py_XDECREF(m_ptrs[i]); + } + m_ptrs = std::move(other.m_ptrs); + } + return *this; + } + + /// Add a pointer, taking ownership (no incref, will decref on destruction) + void push_back_steal(PyObject *p) { m_ptrs.push_back(p); } + + /// Add a pointer, borrowing (increfs now, will decref on destruction) + void push_back_borrow(PyObject *p) { + Py_XINCREF(p); + m_ptrs.push_back(p); + } + + /// Add a null pointer (for PY_VECTORCALL_ARGUMENTS_OFFSET slot) + void push_back_null() { m_ptrs.push_back(nullptr); } + + void reserve(std::size_t sz) { m_ptrs.reserve(sz); } + + std::size_t size() const { return m_ptrs.size(); } + + PyObject *operator[](std::size_t idx) const { return m_ptrs[idx]; } + + PyObject *const *data() const { return m_ptrs.data(); } + +private: + small_vector m_ptrs; +}; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/detail/class.h b/lib/pybind11/include/pybind11/detail/class.h index 528e716..06a761d 100644 --- a/lib/pybind11/include/pybind11/detail/class.h +++ b/lib/pybind11/include/pybind11/detail/class.h @@ -9,8 +9,10 @@ #pragma once -#include "../attr.h" -#include "../options.h" +#include +#include + +#include "exception_translation.h" PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) @@ -69,7 +71,7 @@ inline PyTypeObject *make_static_property_type() { issue no Python C API calls which could potentially invoke the garbage collector (the GC will call type_traverse(), which will in turn find the newly constructed type in an invalid state) */ - auto *heap_type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0); + auto *heap_type = reinterpret_cast(PyType_Type.tp_alloc(&PyType_Type, 0)); if (!heap_type) { pybind11_fail("make_static_property_type(): error allocating type!"); } @@ -86,18 +88,17 @@ inline PyTypeObject *make_static_property_type() { type->tp_descr_get = pybind11_static_get; type->tp_descr_set = pybind11_static_set; - if (PyType_Ready(type) < 0) { - pybind11_fail("make_static_property_type(): failure in PyType_Ready()!"); - } - # if PY_VERSION_HEX >= 0x030C0000 - // PRE 3.12 FEATURE FREEZE. PLEASE REVIEW AFTER FREEZE. // Since Python-3.12 property-derived types are required to // have dynamic attributes (to set `__doc__`) enable_dynamic_attributes(heap_type); # endif - setattr((PyObject *) type, "__module__", str("pybind11_builtins")); + if (PyType_Ready(type) < 0) { + pybind11_fail("make_static_property_type(): failure in PyType_Ready()!"); + } + + setattr(reinterpret_cast(type), "__module__", str(PYBIND11_DUMMY_MODULE_NAME)); PYBIND11_SET_OLDPY_QUALNAME(type, name_obj); return type; @@ -189,12 +190,10 @@ extern "C" inline PyObject *pybind11_meta_call(PyObject *type, PyObject *args, P return nullptr; } - // This must be a pybind11 instance - auto *instance = reinterpret_cast(self); - // Ensure that the base __init__ function(s) were called - for (const auto &vh : values_and_holders(instance)) { - if (!vh.holder_constructed()) { + values_and_holders vhs(self); + for (const auto &vh : vhs) { + if (!vh.holder_constructed() && !vhs.is_redundant_value_and_holder(vh)) { PyErr_Format(PyExc_TypeError, "%.200s.__init__() must be called when overriding __init__", get_fully_qualified_tp_name(vh.type->type).c_str()); @@ -208,39 +207,49 @@ extern "C" inline PyObject *pybind11_meta_call(PyObject *type, PyObject *args, P /// Cleanup the type-info for a pybind11-registered type. extern "C" inline void pybind11_meta_dealloc(PyObject *obj) { - auto *type = (PyTypeObject *) obj; - auto &internals = get_internals(); - - // A pybind11-registered type will: - // 1) be found in internals.registered_types_py - // 2) have exactly one associated `detail::type_info` - auto found_type = internals.registered_types_py.find(type); - if (found_type != internals.registered_types_py.end() && found_type->second.size() == 1 - && found_type->second[0]->type == type) { - - auto *tinfo = found_type->second[0]; - auto tindex = std::type_index(*tinfo->cpptype); - internals.direct_conversions.erase(tindex); - - if (tinfo->module_local) { - get_local_internals().registered_types_cpp.erase(tindex); - } else { - internals.registered_types_cpp.erase(tindex); - } - internals.registered_types_py.erase(tinfo->type); - - // Actually just `std::erase_if`, but that's only available in C++20 - auto &cache = internals.inactive_override_cache; - for (auto it = cache.begin(), last = cache.end(); it != last;) { - if (it->first == (PyObject *) tinfo->type) { - it = cache.erase(it); + with_internals_if_internals([obj](internals &internals) { + auto *type = (PyTypeObject *) obj; + + // A pybind11-registered type will: + // 1) be found in internals.registered_types_py + // 2) have exactly one associated `detail::type_info` + auto found_type = internals.registered_types_py.find(type); + if (found_type != internals.registered_types_py.end() && found_type->second.size() == 1 + && found_type->second[0]->type == type) { + + auto *tinfo = found_type->second[0]; + auto tindex = std::type_index(*tinfo->cpptype); + internals.direct_conversions.erase(tindex); + + auto &local_internals = get_local_internals(); + if (tinfo->module_local) { + local_internals.registered_types_cpp.erase(tinfo->cpptype); } else { - ++it; + internals.registered_types_cpp.erase(tindex); +#if PYBIND11_INTERNALS_VERSION >= 12 + internals.registered_types_cpp_fast.erase(tinfo->cpptype); + for (const std::type_info *alias : tinfo->alias_chain) { + auto num_erased = internals.registered_types_cpp_fast.erase(alias); + (void) num_erased; + assert(num_erased > 0); + } +#endif + } + internals.registered_types_py.erase(tinfo->type); + + // Actually just `std::erase_if`, but that's only available in C++20 + auto &cache = internals.inactive_override_cache; + for (auto it = cache.begin(), last = cache.end(); it != last;) { + if (it->first == (PyObject *) tinfo->type) { + it = cache.erase(it); + } else { + ++it; + } } - } - delete tinfo; - } + delete tinfo; + } + }); PyType_Type.tp_dealloc(obj); } @@ -256,7 +265,7 @@ inline PyTypeObject *make_default_metaclass() { issue no Python C API calls which could potentially invoke the garbage collector (the GC will call type_traverse(), which will in turn find the newly constructed type in an invalid state) */ - auto *heap_type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0); + auto *heap_type = reinterpret_cast(PyType_Type.tp_alloc(&PyType_Type, 0)); if (!heap_type) { pybind11_fail("make_default_metaclass(): error allocating metaclass!"); } @@ -282,7 +291,7 @@ inline PyTypeObject *make_default_metaclass() { pybind11_fail("make_default_metaclass(): failure in PyType_Ready()!"); } - setattr((PyObject *) type, "__module__", str("pybind11_builtins")); + setattr(reinterpret_cast(type), "__module__", str(PYBIND11_DUMMY_MODULE_NAME)); PYBIND11_SET_OLDPY_QUALNAME(type, name_obj); return type; @@ -297,7 +306,7 @@ inline void traverse_offset_bases(void *valueptr, instance *self, bool (*f)(void * /*parentptr*/, instance * /*self*/)) { for (handle h : reinterpret_borrow(tinfo->type->tp_bases)) { - if (auto *parent_tinfo = get_type_info((PyTypeObject *) h.ptr())) { + if (auto *parent_tinfo = get_type_info(reinterpret_cast(h.ptr()))) { for (auto &c : parent_tinfo->implicit_casts) { if (c.first == tinfo->cpptype) { auto *parentptr = c.second(valueptr); @@ -312,20 +321,49 @@ inline void traverse_offset_bases(void *valueptr, } } +#ifdef Py_GIL_DISABLED +inline void enable_try_inc_ref(PyObject *obj) { +# if PY_VERSION_HEX >= 0x030E00A4 + PyUnstable_EnableTryIncRef(obj); +# else + if (_Py_IsImmortal(obj)) { + return; + } + for (;;) { + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&obj->ob_ref_shared); + if ((shared & _Py_REF_SHARED_FLAG_MASK) != 0) { + // Nothing to do if it's in WEAKREFS, QUEUED, or MERGED states. + return; + } + if (_Py_atomic_compare_exchange_ssize( + &obj->ob_ref_shared, &shared, shared | _Py_REF_MAYBE_WEAKREF)) { + return; + } + } +# endif +} +#endif + inline bool register_instance_impl(void *ptr, instance *self) { - get_internals().registered_instances.emplace(ptr, self); + assert(ptr); +#ifdef Py_GIL_DISABLED + enable_try_inc_ref(reinterpret_cast(self)); +#endif + with_instance_map(ptr, [&](instance_map &instances) { instances.emplace(ptr, self); }); return true; // unused, but gives the same signature as the deregister func } inline bool deregister_instance_impl(void *ptr, instance *self) { - auto ®istered_instances = get_internals().registered_instances; - auto range = registered_instances.equal_range(ptr); - for (auto it = range.first; it != range.second; ++it) { - if (self == it->second) { - registered_instances.erase(it); - return true; + assert(ptr); + return with_instance_map(ptr, [&](instance_map &instances) { + auto range = instances.equal_range(ptr); + for (auto it = range.first; it != range.second; ++it) { + if (self == it->second) { + instances.erase(it); + return true; + } } - } - return false; + return false; + }); } inline void register_instance(instance *self, void *valptr, const type_info *tinfo) { @@ -375,28 +413,37 @@ extern "C" inline PyObject *pybind11_object_new(PyTypeObject *type, PyObject *, extern "C" inline int pybind11_object_init(PyObject *self, PyObject *, PyObject *) { PyTypeObject *type = Py_TYPE(self); std::string msg = get_fully_qualified_tp_name(type) + ": No constructor defined!"; - PyErr_SetString(PyExc_TypeError, msg.c_str()); + set_error(PyExc_TypeError, msg.c_str()); return -1; } inline void add_patient(PyObject *nurse, PyObject *patient) { - auto &internals = get_internals(); auto *instance = reinterpret_cast(nurse); instance->has_patients = true; Py_INCREF(patient); - internals.patients[nurse].push_back(patient); + + with_internals([&](internals &internals) { internals.patients[nurse].push_back(patient); }); } inline void clear_patients(PyObject *self) { auto *instance = reinterpret_cast(self); - auto &internals = get_internals(); - auto pos = internals.patients.find(self); - assert(pos != internals.patients.end()); - // Clearing the patients can cause more Python code to run, which - // can invalidate the iterator. Extract the vector of patients - // from the unordered_map first. - auto patients = std::move(pos->second); - internals.patients.erase(pos); + std::vector patients; + + with_internals([&](internals &internals) { + auto pos = internals.patients.find(self); + + if (pos == internals.patients.end()) { + pybind11_fail( + "FATAL: Internal consistency check failed: Invalid clear_patients() call."); + } + + // Clearing the patients can cause more Python code to run, which + // can invalidate the iterator. Extract the vector of patients + // from the unordered_map first. + patients = std::move(pos->second); + internals.patients.erase(pos); + }); + instance->has_patients = false; for (PyObject *&patient : patients) { Py_CLEAR(patient); @@ -423,6 +470,8 @@ inline void clear_instance(PyObject *self) { if (instance->owned || v_h.holder_constructed()) { v_h.type->dealloc(v_h); } + } else if (v_h.holder_constructed()) { + v_h.type->dealloc(v_h); // Disowned instance. } } // Deallocate the value/holder layout internals: @@ -445,28 +494,42 @@ inline void clear_instance(PyObject *self) { /// Instance destructor function for all pybind11 types. It calls `type_info.dealloc` /// to destroy the C++ object itself, while the rest is Python bookkeeping. extern "C" inline void pybind11_object_dealloc(PyObject *self) { + auto *type = Py_TYPE(self); + + // If this is a GC tracked object, untrack it first + // Note that the track call is implicitly done by the + // default tp_alloc, which we never override. + if (PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC) != 0) { + PyObject_GC_UnTrack(self); + } + +#if PY_VERSION_HEX >= 0x030D0000 + // PyObject_ClearManagedDict() is available from Python 3.13+. It must be + // called before tp_free() because on Python 3.14+ tp_free no longer + // implicitly clears the managed dict, which would abandon the refcounts of + // objects stored in __dict__ of py::dynamic_attr() types, causing permanent + // memory leaks. + if (PyType_HasFeature(type, Py_TPFLAGS_MANAGED_DICT)) { + PyObject_ClearManagedDict(self); + } +#endif + clear_instance(self); - auto *type = Py_TYPE(self); type->tp_free(self); -#if PY_VERSION_HEX < 0x03080000 - // `type->tp_dealloc != pybind11_object_dealloc` means that we're being called - // as part of a derived type's dealloc, in which case we're not allowed to decref - // the type here. For cross-module compatibility, we shouldn't compare directly - // with `pybind11_object_dealloc`, but with the common one stashed in internals. - auto pybind11_object_type = (PyTypeObject *) get_internals().instance_base; - if (type->tp_dealloc == pybind11_object_type->tp_dealloc) - Py_DECREF(type); -#else // This was not needed before Python 3.8 (Python issue 35810) // https://github.com/pybind/pybind11/issues/1946 Py_DECREF(type); -#endif } +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_GCC("-Wredundant-decls") + std::string error_string(); +PYBIND11_WARNING_POP + /** Create the type which can be used as a common base for all classes. This is needed in order to satisfy Python's requirements for multiple inheritance. Return value: New reference. */ @@ -478,7 +541,7 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass) { issue no Python C API calls which could potentially invoke the garbage collector (the GC will call type_traverse(), which will in turn find the newly constructed type in an invalid state) */ - auto *heap_type = (PyHeapTypeObject *) metaclass->tp_alloc(metaclass, 0); + auto *heap_type = reinterpret_cast(metaclass->tp_alloc(metaclass, 0)); if (!heap_type) { pybind11_fail("make_object_base_type(): error allocating type!"); } @@ -505,28 +568,37 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass) { pybind11_fail("PyType_Ready failed in make_object_base_type(): " + error_string()); } - setattr((PyObject *) type, "__module__", str("pybind11_builtins")); + setattr(reinterpret_cast(type), "__module__", str(PYBIND11_DUMMY_MODULE_NAME)); PYBIND11_SET_OLDPY_QUALNAME(type, name_obj); assert(!PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC)); - return (PyObject *) heap_type; + return reinterpret_cast(heap_type); } /// dynamic_attr: Allow the garbage collector to traverse the internal instance `__dict__`. extern "C" inline int pybind11_traverse(PyObject *self, visitproc visit, void *arg) { +#if PY_VERSION_HEX >= 0x030D0000 + int ret = PyObject_VisitManagedDict(self, visit, arg); + if (ret) { + return ret; + } +#else PyObject *&dict = *_PyObject_GetDictPtr(self); Py_VISIT(dict); -// https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_traverse -#if PY_VERSION_HEX >= 0x03090000 - Py_VISIT(Py_TYPE(self)); #endif + // https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_traverse + Py_VISIT(Py_TYPE(self)); return 0; } /// dynamic_attr: Allow the GC to clear the dictionary. extern "C" inline int pybind11_clear(PyObject *self) { +#if PY_VERSION_HEX >= 0x030D0000 + PyObject_ClearManagedDict(self); +#else PyObject *&dict = *_PyObject_GetDictPtr(self); Py_CLEAR(dict); +#endif return 0; } @@ -534,7 +606,7 @@ extern "C" inline int pybind11_clear(PyObject *self) { inline void enable_dynamic_attributes(PyHeapTypeObject *heap_type) { auto *type = &heap_type->ht_type; type->tp_flags |= Py_TPFLAGS_HAVE_GC; -#if PY_VERSION_HEX < 0x030B0000 +#ifdef PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET type->tp_dictoffset = type->tp_basicsize; // place dict at the end type->tp_basicsize += (ssize_t) sizeof(PyObject *); // and allocate enough space for it #else @@ -543,17 +615,9 @@ inline void enable_dynamic_attributes(PyHeapTypeObject *heap_type) { type->tp_traverse = pybind11_traverse; type->tp_clear = pybind11_clear; - static PyGetSetDef getset[] = {{ -#if PY_VERSION_HEX < 0x03070000 - const_cast("__dict__"), -#else - "__dict__", -#endif - PyObject_GenericGetDict, - PyObject_GenericSetDict, - nullptr, - nullptr}, - {nullptr, nullptr, nullptr, nullptr, nullptr}}; + static PyGetSetDef getset[] + = {{"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, nullptr, nullptr}, + {nullptr, nullptr, nullptr, nullptr, nullptr}}; type->tp_getset = getset; } @@ -571,35 +635,89 @@ extern "C" inline int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int fla if (view) { view->obj = nullptr; } - PyErr_SetString(PyExc_BufferError, "pybind11_getbuffer(): Internal error"); + set_error(PyExc_BufferError, "pybind11_getbuffer(): Internal error"); return -1; } std::memset(view, 0, sizeof(Py_buffer)); - buffer_info *info = tinfo->get_buffer(obj, tinfo->get_buffer_data); + std::unique_ptr info = nullptr; + try { + info.reset(tinfo->get_buffer(obj, tinfo->get_buffer_data)); + } catch (...) { + try_translate_exceptions(); + raise_from(PyExc_BufferError, "Error getting buffer"); + return -1; + } + if (info == nullptr) { + pybind11_fail("FATAL UNEXPECTED SITUATION: tinfo->get_buffer() returned nullptr."); + } + if ((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE && info->readonly) { - delete info; // view->obj = nullptr; // Was just memset to 0, so not necessary - PyErr_SetString(PyExc_BufferError, "Writable buffer requested for readonly storage"); + set_error(PyExc_BufferError, "Writable buffer requested for readonly storage"); return -1; } - view->obj = obj; - view->ndim = 1; - view->internal = info; - view->buf = info->ptr; + + // Fill in all the information, and then downgrade as requested by the caller, or raise an + // error if that's not possible. view->itemsize = info->itemsize; view->len = view->itemsize; for (auto s : info->shape) { view->len *= s; } + view->ndim = static_cast(info->ndim); + view->shape = info->shape.data(); + view->strides = info->strides.data(); view->readonly = static_cast(info->readonly); if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) { view->format = const_cast(info->format.c_str()); } - if ((flags & PyBUF_STRIDES) == PyBUF_STRIDES) { - view->ndim = (int) info->ndim; - view->strides = info->strides.data(); - view->shape = info->shape.data(); + + // Note, all contiguity flags imply PyBUF_STRIDES and lower. + if ((flags & PyBUF_C_CONTIGUOUS) == PyBUF_C_CONTIGUOUS) { + if (PyBuffer_IsContiguous(view, 'C') == 0) { + std::memset(view, 0, sizeof(Py_buffer)); + set_error(PyExc_BufferError, + "C-contiguous buffer requested for discontiguous storage"); + return -1; + } + } else if ((flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS) { + if (PyBuffer_IsContiguous(view, 'F') == 0) { + std::memset(view, 0, sizeof(Py_buffer)); + set_error(PyExc_BufferError, + "Fortran-contiguous buffer requested for discontiguous storage"); + return -1; + } + } else if ((flags & PyBUF_ANY_CONTIGUOUS) == PyBUF_ANY_CONTIGUOUS) { + if (PyBuffer_IsContiguous(view, 'A') == 0) { + std::memset(view, 0, sizeof(Py_buffer)); + set_error(PyExc_BufferError, "Contiguous buffer requested for discontiguous storage"); + return -1; + } + + } else if ((flags & PyBUF_STRIDES) != PyBUF_STRIDES) { + // If no strides are requested, the buffer must be C-contiguous. + // https://docs.python.org/3/c-api/buffer.html#contiguity-requests + if (PyBuffer_IsContiguous(view, 'C') == 0) { + std::memset(view, 0, sizeof(Py_buffer)); + set_error(PyExc_BufferError, + "C-contiguous buffer requested for discontiguous storage"); + return -1; + } + + view->strides = nullptr; + + // Since this is a contiguous buffer, it can also pretend to be 1D. + if ((flags & PyBUF_ND) != PyBUF_ND) { + view->shape = nullptr; + view->ndim = 0; + } } + + // Set these after all checks so they don't leak out into the caller, and can be automatically + // cleaned up on error. + view->buf = info->ptr; + view->internal = info.release(); + view->obj = obj; Py_INCREF(view->obj); return 0; } @@ -628,15 +746,7 @@ inline PyObject *make_new_python_type(const type_record &rec) { PyUnicode_FromFormat("%U.%U", rec.scope.attr("__qualname__").ptr(), name.ptr())); } - object module_; - if (rec.scope) { - if (hasattr(rec.scope, "__module__")) { - module_ = rec.scope.attr("__module__"); - } else if (hasattr(rec.scope, "__name__")) { - module_ = rec.scope.attr("__name__"); - } - } - + object module_ = get_module_name_if_available(rec.scope); const auto *full_name = c_str( #if !defined(PYPY_VERSION) module_ ? str(module_).cast() + "." + rec.name : @@ -645,10 +755,13 @@ inline PyObject *make_new_python_type(const type_record &rec) { char *tp_doc = nullptr; if (rec.doc && options::show_user_defined_docstrings()) { - /* Allocate memory for docstring (using PyObject_MALLOC, since - Python will free this later on) */ + /* Allocate memory for docstring (Python will free this later on) */ size_t size = std::strlen(rec.doc) + 1; +#if PY_VERSION_HEX >= 0x030D0000 + tp_doc = static_cast(PyMem_MALLOC(size)); +#else tp_doc = (char *) PyObject_MALLOC(size); +#endif std::memcpy((void *) tp_doc, rec.doc, size); } @@ -660,10 +773,10 @@ inline PyObject *make_new_python_type(const type_record &rec) { issue no Python C API calls which could potentially invoke the garbage collector (the GC will call type_traverse(), which will in turn find the newly constructed type in an invalid state) */ - auto *metaclass - = rec.metaclass.ptr() ? (PyTypeObject *) rec.metaclass.ptr() : internals.default_metaclass; + auto *metaclass = rec.metaclass.ptr() ? reinterpret_cast(rec.metaclass.ptr()) + : internals.default_metaclass; - auto *heap_type = (PyHeapTypeObject *) metaclass->tp_alloc(metaclass, 0); + auto *heap_type = reinterpret_cast(metaclass->tp_alloc(metaclass, 0)); if (!heap_type) { pybind11_fail(std::string(rec.name) + ": Unable to create type object!"); } @@ -676,7 +789,7 @@ inline PyObject *make_new_python_type(const type_record &rec) { auto *type = &heap_type->ht_type; type->tp_name = full_name; type->tp_doc = tp_doc; - type->tp_base = type_incref((PyTypeObject *) base); + type->tp_base = type_incref(reinterpret_cast(base)); type->tp_basicsize = static_cast(sizeof(instance)); if (!bases.empty()) { type->tp_bases = bases.release().ptr(); @@ -717,18 +830,18 @@ inline PyObject *make_new_python_type(const type_record &rec) { /* Register type with the parent scope */ if (rec.scope) { - setattr(rec.scope, rec.name, (PyObject *) type); + setattr(rec.scope, rec.name, reinterpret_cast(type)); } else { Py_INCREF(type); // Keep it alive forever (reference leak) } if (module_) { // Needed by pydoc - setattr((PyObject *) type, "__module__", module_); + setattr(reinterpret_cast(type), "__module__", module_); } PYBIND11_SET_OLDPY_QUALNAME(type, qualname); - return (PyObject *) type; + return reinterpret_cast(type); } PYBIND11_NAMESPACE_END(detail) diff --git a/lib/pybind11/include/pybind11/detail/common.h b/lib/pybind11/include/pybind11/detail/common.h index 0fc81e0..740001d 100644 --- a/lib/pybind11/include/pybind11/detail/common.h +++ b/lib/pybind11/include/pybind11/detail/common.h @@ -9,88 +9,42 @@ #pragma once -#define PYBIND11_VERSION_MAJOR 2 -#define PYBIND11_VERSION_MINOR 11 -#define PYBIND11_VERSION_PATCH 0.dev1 - -// Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html -// Additional convention: 0xD = dev -#define PYBIND11_VERSION_HEX 0x020B00D1 - -// Define some generic pybind11 helper macros for warning management. -// -// Note that compiler-specific push/pop pairs are baked into the -// PYBIND11_NAMESPACE_BEGIN/PYBIND11_NAMESPACE_END pair of macros. Therefore manual -// PYBIND11_WARNING_PUSH/PYBIND11_WARNING_POP are usually only needed in `#include` sections. -// -// If you find you need to suppress a warning, please try to make the suppression as local as -// possible using these macros. Please also be sure to push/pop with the pybind11 macros. Please -// only use compiler specifics if you need to check specific versions, e.g. Apple Clang vs. vanilla -// Clang. -#if defined(_MSC_VER) -# define PYBIND11_COMPILER_MSVC -# define PYBIND11_PRAGMA(...) __pragma(__VA_ARGS__) -# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(warning(push)) -# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(warning(pop)) -#elif defined(__INTEL_COMPILER) -# define PYBIND11_COMPILER_INTEL -# define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__) -# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(warning push) -# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(warning pop) -#elif defined(__clang__) -# define PYBIND11_COMPILER_CLANG -# define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__) -# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(clang diagnostic push) -# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(clang diagnostic push) -#elif defined(__GNUC__) -# define PYBIND11_COMPILER_GCC -# define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__) -# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(GCC diagnostic push) -# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(GCC diagnostic pop) -#endif - -#ifdef PYBIND11_COMPILER_MSVC -# define PYBIND11_WARNING_DISABLE_MSVC(name) PYBIND11_PRAGMA(warning(disable : name)) -#else -# define PYBIND11_WARNING_DISABLE_MSVC(name) +#include +#if PY_VERSION_HEX < 0x03090000 +# error "PYTHON < 3.9 IS UNSUPPORTED. pybind11 v3.0 was the last to support Python 3.8." #endif -#ifdef PYBIND11_COMPILER_CLANG -# define PYBIND11_WARNING_DISABLE_CLANG(name) PYBIND11_PRAGMA(clang diagnostic ignored name) -#else -# define PYBIND11_WARNING_DISABLE_CLANG(name) -#endif - -#ifdef PYBIND11_COMPILER_GCC -# define PYBIND11_WARNING_DISABLE_GCC(name) PYBIND11_PRAGMA(GCC diagnostic ignored name) -#else -# define PYBIND11_WARNING_DISABLE_GCC(name) -#endif - -#ifdef PYBIND11_COMPILER_INTEL -# define PYBIND11_WARNING_DISABLE_INTEL(name) PYBIND11_PRAGMA(warning disable name) -#else -# define PYBIND11_WARNING_DISABLE_INTEL(name) +// Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html +// See also: https://github.com/python/cpython/blob/HEAD/Include/patchlevel.h +/* -- start version constants -- */ +#define PYBIND11_VERSION_MAJOR 3 +#define PYBIND11_VERSION_MINOR 1 +#define PYBIND11_VERSION_MICRO 0 +// ALPHA = 0xA, BETA = 0xB, GAMMA = 0xC (release candidate), FINAL = 0xF (stable release) +// - The release level is set to "alpha" for development versions. +// Use 0xA0 (LEVEL=0xA, SERIAL=0) for development versions. +// - For stable releases, set the serial to 0. +#define PYBIND11_VERSION_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL +#define PYBIND11_VERSION_RELEASE_SERIAL 0 +// String version of (micro, release level, release serial), e.g.: 0a0, 0b1, 0rc1, 0 +#define PYBIND11_VERSION_PATCH 0 +/* -- end version constants -- */ + +#if !defined(Py_PACK_FULL_VERSION) +// Stable API since Python 3.14.0a4 +# define Py_PACK_FULL_VERSION(X, Y, Z, LEVEL, SERIAL) \ + ((((X) & 0xff) << 24) | (((Y) & 0xff) << 16) | (((Z) & 0xff) << 8) \ + | (((LEVEL) & 0xf) << 4) | (((SERIAL) & 0xf) << 0)) #endif +// Version as a single 4-byte hex number, e.g. 0x030C04B5 == 3.12.4b5. +#define PYBIND11_VERSION_HEX \ + Py_PACK_FULL_VERSION(PYBIND11_VERSION_MAJOR, \ + PYBIND11_VERSION_MINOR, \ + PYBIND11_VERSION_MICRO, \ + PYBIND11_VERSION_RELEASE_LEVEL, \ + PYBIND11_VERSION_RELEASE_SERIAL) -#define PYBIND11_NAMESPACE_BEGIN(name) \ - namespace name { \ - PYBIND11_WARNING_PUSH - -#define PYBIND11_NAMESPACE_END(name) \ - PYBIND11_WARNING_POP \ - } - -// Robust support for some features and loading modules compiled against different pybind versions -// requires forcing hidden visibility on pybind code, so we enforce this by setting the attribute -// on the main `pybind11` namespace. -#if !defined(PYBIND11_NAMESPACE) -# ifdef __GNUG__ -# define PYBIND11_NAMESPACE pybind11 __attribute__((visibility("hidden"))) -# else -# define PYBIND11_NAMESPACE pybind11 -# endif -#endif +#include "pybind11_namespace_macros.h" #if !(defined(_MSC_VER) && __cplusplus == 199711L) # if __cplusplus >= 201402L @@ -118,6 +72,41 @@ # endif #endif +// These PYBIND11_HAS_... macros are consolidated in pybind11/detail/common.h +// to simplify backward compatibility handling for users (e.g., via #ifdef checks): +#define PYBIND11_HAS_TYPE_CASTER_STD_FUNCTION_SPECIALIZATIONS 1 +#define PYBIND11_HAS_INTERNALS_WITH_SMART_HOLDER_SUPPORT 1 +#define PYBIND11_HAS_CPP_CONDUIT 1 +#define PYBIND11_HAS_NATIVE_ENUM 1 + +#if defined(PYBIND11_CPP17) && defined(__has_include) +# if __has_include() +# define PYBIND11_HAS_FILESYSTEM 1 +# elif __has_include() +# define PYBIND11_HAS_EXPERIMENTAL_FILESYSTEM 1 +# endif +#endif + +#if defined(__cpp_lib_launder) +# define PYBIND11_STD_LAUNDER std::launder +# define PYBIND11_HAS_STD_LAUNDER 1 +#else +# define PYBIND11_STD_LAUNDER +# define PYBIND11_HAS_STD_LAUNDER 0 +#endif + +#if defined(PYBIND11_CPP20) +# define PYBIND11_CONSTINIT constinit +# define PYBIND11_DTOR_CONSTEXPR constexpr +#else +# define PYBIND11_CONSTINIT +# define PYBIND11_DTOR_CONSTEXPR +#endif + +#if defined(PYBIND11_CPP20) && defined(__has_include) && __has_include() +# define PYBIND11_HAS_STD_BARRIER 1 +#endif + // Compiler version assertions #if defined(__INTEL_COMPILER) # if __INTEL_COMPILER < 1800 @@ -143,8 +132,8 @@ # error pybind11 requires gcc 4.8 or newer # endif #elif defined(_MSC_VER) -# if _MSC_VER < 1910 -# error pybind11 2.10+ requires MSVC 2017 or newer +# if _MSC_VER < 1920 +# error pybind11 3.1+ requires MSVC 2019 or newer # endif #endif @@ -156,14 +145,6 @@ # endif #endif -#if !defined(PYBIND11_EXPORT_EXCEPTION) -# if defined(__apple_build_version__) -# define PYBIND11_EXPORT_EXCEPTION PYBIND11_EXPORT -# else -# define PYBIND11_EXPORT_EXCEPTION -# endif -#endif - // For CUDA, GCC7, GCC8: // PYBIND11_NOINLINE_FORCED is incompatible with `-Wattributes -Werror`. // When defining PYBIND11_NOINLINE_FORCED, it is best to also use `-Wno-attributes`. @@ -186,6 +167,14 @@ # define PYBIND11_NOINLINE __attribute__((noinline)) inline #endif +#if defined(_MSC_VER) +# define PYBIND11_ALWAYS_INLINE __forceinline +#elif defined(__GNUC__) +# define PYBIND11_ALWAYS_INLINE __attribute__((__always_inline__)) inline +#else +# define PYBIND11_ALWAYS_INLINE inline +#endif + #if defined(__MINGW32__) // For unknown reasons all PYBIND11_DEPRECATED member trigger a warning when declared // whether it is used or not @@ -204,31 +193,6 @@ # define PYBIND11_MAYBE_UNUSED __attribute__((__unused__)) #endif -/* Don't let Python.h #define (v)snprintf as macro because they are implemented - properly in Visual Studio since 2015. */ -#if defined(_MSC_VER) -# define HAVE_SNPRINTF 1 -#endif - -/// Include Python header, disable linking to pythonX_d.lib on Windows in debug mode -#if defined(_MSC_VER) -PYBIND11_WARNING_PUSH -PYBIND11_WARNING_DISABLE_MSVC(4505) -// C4505: 'PySlice_GetIndicesEx': unreferenced local function has been removed (PyPy only) -# if defined(_DEBUG) && !defined(Py_DEBUG) -// Workaround for a VS 2022 issue. -// NOTE: This workaround knowingly violates the Python.h include order requirement: -// https://docs.python.org/3/c-api/intro.html#include-files -// See https://github.com/pybind/pybind11/pull/3497 for full context. -# include -# if _MSVC_STL_VERSION >= 143 -# include -# endif -# define PYBIND11_DEBUG_MARKER -# undef _DEBUG -# endif -#endif - // https://en.cppreference.com/w/c/chrono/localtime #if defined(__STDC_LIB_EXT1__) && !defined(__STDC_WANT_LIB_EXT1__) # define __STDC_WANT_LIB_EXT1__ @@ -253,53 +217,15 @@ PYBIND11_WARNING_DISABLE_MSVC(4505) # define PYBIND11_HAS_VARIANT 1 #endif -#if defined(PYBIND11_CPP17) -# if defined(__has_include) -# if __has_include() -# define PYBIND11_HAS_STRING_VIEW -# endif -# elif defined(_MSC_VER) -# define PYBIND11_HAS_STRING_VIEW -# endif +#if defined(PYBIND11_CPP17) \ + && ((defined(__has_include) && __has_include()) || defined(_MSC_VER)) +# define PYBIND11_HAS_STRING_VIEW 1 #endif -#include -// Reminder: WITH_THREAD is always defined if PY_VERSION_HEX >= 0x03070000 -#if PY_VERSION_HEX < 0x03060000 -# error "PYTHON < 3.6 IS UNSUPPORTED. pybind11 v2.9 was the last to support Python 2 and 3.5." -#endif -#include -#include - -/* Python #defines overrides on all sorts of core functions, which - tends to weak havok in C++ codebases that expect these to work - like regular functions (potentially with several overloads) */ -#if defined(isalnum) -# undef isalnum -# undef isalpha -# undef islower -# undef isspace -# undef isupper -# undef tolower -# undef toupper -#endif - -#if defined(copysign) -# undef copysign -#endif - -#if defined(PYPY_VERSION) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +#if (defined(PYPY_VERSION) || defined(GRAALVM_PYTHON)) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) # define PYBIND11_SIMPLE_GIL_MANAGEMENT #endif -#if defined(_MSC_VER) -# if defined(PYBIND11_DEBUG_MARKER) -# define _DEBUG -# undef PYBIND11_DEBUG_MARKER -# endif -PYBIND11_WARNING_POP -#endif - #include #include #include @@ -318,20 +244,76 @@ PYBIND11_WARNING_POP # endif #endif +// For libc++, the exceptions should be exported, +// otherwise, the exception translation would be incorrect. +// IMPORTANT: This code block must stay BELOW the #include above (see PR #5390). +#if !defined(PYBIND11_EXPORT_EXCEPTION) +# if defined(_LIBCPP_EXCEPTION) +# define PYBIND11_EXPORT_EXCEPTION PYBIND11_EXPORT +# else +# define PYBIND11_EXPORT_EXCEPTION +# endif +#endif + // Must be after including or one of the other headers specified by the standard #if defined(__cpp_lib_char8_t) && __cpp_lib_char8_t >= 201811L -# define PYBIND11_HAS_U8STRING +# define PYBIND11_HAS_U8STRING 1 +#endif + +#if defined(PYBIND11_CPP20) && defined(__cpp_lib_span) && __cpp_lib_span >= 202002L +# define PYBIND11_HAS_SPAN 1 #endif // See description of PR #4246: -#if !defined(NDEBUG) && !defined(PY_ASSERT_GIL_HELD_INCREF_DECREF) \ - && !(defined(PYPY_VERSION) \ - && defined(_MSC_VER)) /* PyPy Windows: pytest hangs indefinitely at the end of the \ - process (see PR #4268) */ \ - && !defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) +#if !defined(PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF) && !defined(NDEBUG) \ + && !defined(PYPY_VERSION) && !defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) # define PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF #endif +// Slightly faster code paths are available when PYBIND11_HAS_SUBINTERPRETER_SUPPORT is *not* +// defined, so avoid defining it for implementations that do not support subinterpreters. However, +// defining it unnecessarily is not expected to break anything. +// This can be overridden by the user with -DPYBIND11_HAS_SUBINTERPRETER_SUPPORT=1 or 0 +#ifndef PYBIND11_HAS_SUBINTERPRETER_SUPPORT +# if PY_VERSION_HEX >= 0x030C0000 && !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) +# define PYBIND11_HAS_SUBINTERPRETER_SUPPORT 1 +# endif +#else +# if PYBIND11_HAS_SUBINTERPRETER_SUPPORT == 0 +# undef PYBIND11_HAS_SUBINTERPRETER_SUPPORT +# endif +#endif + +// 3.13 Compatibility +#if 0x030D0000 <= PY_VERSION_HEX +# define PYBIND11_TYPE_IS_TYPE_HINT "typing.TypeIs" +# define PYBIND11_CAPSULE_TYPE_TYPE_HINT "types.CapsuleType" +#else +# define PYBIND11_TYPE_IS_TYPE_HINT "typing_extensions.TypeIs" +# define PYBIND11_CAPSULE_TYPE_TYPE_HINT "typing_extensions.CapsuleType" +#endif + +// 3.12 Compatibility +#if 0x030C0000 <= PY_VERSION_HEX +# define PYBIND11_BUFFER_TYPE_HINT "collections.abc.Buffer" +#else +# define PYBIND11_BUFFER_TYPE_HINT "typing_extensions.Buffer" +#endif + +// 3.11 Compatibility +#if 0x030B0000 <= PY_VERSION_HEX +# define PYBIND11_NEVER_TYPE_HINT "typing.Never" +#else +# define PYBIND11_NEVER_TYPE_HINT "typing_extensions.Never" +#endif + +// 3.10 Compatibility +#if 0x030A0000 <= PY_VERSION_HEX +# define PYBIND11_TYPE_GUARD_TYPE_HINT "typing.TypeGuard" +#else +# define PYBIND11_TYPE_GUARD_TYPE_HINT "typing_extensions.TypeGuard" +#endif + // #define PYBIND11_STR_LEGACY_PERMISSIVE // If DEFINED, pybind11::str can hold PyUnicodeObject or PyBytesObject // (probably surprising and never documented, but this was the @@ -356,6 +338,13 @@ PYBIND11_WARNING_POP #define PYBIND11_BYTES_AS_STRING PyBytes_AsString #define PYBIND11_BYTES_SIZE PyBytes_Size #define PYBIND11_LONG_CHECK(o) PyLong_Check(o) +// In PyPy 7.3.3, `PyIndex_Check` is implemented by calling `__index__`, +// while CPython only considers the existence of `nb_index`/`__index__`. +#if !defined(PYPY_VERSION) +# define PYBIND11_INDEX_CHECK(o) PyIndex_Check(o) +#else +# define PYBIND11_INDEX_CHECK(o) hasattr(o, "__index__") +#endif #define PYBIND11_LONG_AS_LONGLONG(o) PyLong_AsLongLong(o) #define PYBIND11_LONG_FROM_SIGNED(o) PyLong_FromSsize_t((ssize_t) (o)) #define PYBIND11_LONG_FROM_UNSIGNED(o) PyLong_FromSize_t((size_t) (o)) @@ -369,15 +358,35 @@ PYBIND11_WARNING_POP #define PYBIND11_BUILTINS_MODULE "builtins" // Providing a separate declaration to make Clang's -Wmissing-prototypes happy. // See comment for PYBIND11_MODULE below for why this is marked "maybe unused". +#define PYBIND11_PLUGIN_DECL(name) \ + extern "C" PYBIND11_MAYBE_UNUSED PYBIND11_EXPORT PyObject *PyInit_##name(); #define PYBIND11_PLUGIN_IMPL(name) \ - extern "C" PYBIND11_MAYBE_UNUSED PYBIND11_EXPORT PyObject *PyInit_##name(); \ + PYBIND11_PLUGIN_DECL(name) \ extern "C" PYBIND11_EXPORT PyObject *PyInit_##name() #define PYBIND11_TRY_NEXT_OVERLOAD ((PyObject *) 1) // special failure return code #define PYBIND11_STRINGIFY(x) #x #define PYBIND11_TOSTRING(x) PYBIND11_STRINGIFY(x) #define PYBIND11_CONCAT(first, second) first##second -#define PYBIND11_ENSURE_INTERNALS_READY pybind11::detail::get_internals(); +#define PYBIND11_ENSURE_INTERNALS_READY \ + { \ + pybind11::detail::get_internals_pp_manager().unref(); \ + pybind11::detail::get_internals(); \ + } + +#if !defined(GRAALVM_PYTHON) +# define PYBIND11_PYCFUNCTION_GET_DOC(func) ((func)->m_ml->ml_doc) +# define PYBIND11_PYCFUNCTION_SET_DOC(func, doc) \ + do { \ + (func)->m_ml->ml_doc = (doc); \ + } while (0) +#else +# define PYBIND11_PYCFUNCTION_GET_DOC(func) (GraalPyCFunction_GetDoc((PyObject *) (func))) +# define PYBIND11_PYCFUNCTION_SET_DOC(func, doc) \ + do { \ + GraalPyCFunction_SetDoc((PyObject *) (func), (doc)); \ + } while (0) +#endif #define PYBIND11_CHECK_PYTHON_VERSION \ { \ @@ -399,11 +408,9 @@ PYBIND11_WARNING_POP #define PYBIND11_CATCH_INIT_EXCEPTIONS \ catch (pybind11::error_already_set & e) { \ pybind11::raise_from(e, PyExc_ImportError, "initialization failed"); \ - return nullptr; \ } \ catch (const std::exception &e) { \ - PyErr_SetString(PyExc_ImportError, e.what()); \ - return nullptr; \ + ::pybind11::set_error(PyExc_ImportError, e.what()); \ } /** \rst @@ -431,14 +438,68 @@ PYBIND11_WARNING_POP return pybind11_init(); \ } \ PYBIND11_CATCH_INIT_EXCEPTIONS \ + return nullptr; \ } \ PyObject *pybind11_init() +// this push is for the next several macros +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_CLANG("-Wgnu-zero-variadic-macro-arguments") + +/** +Create a PyInit_ function for this module. + +Note that this is run once for each (sub-)interpreter the module is imported into, including +possibly concurrently. The PyModuleDef is allowed to be static, but the PyObject* resulting from +PyModuleDef_Init should be treated like any other PyObject (so not shared across interpreters). + */ +#define PYBIND11_MODULE_PYINIT(name, ...) \ + static int PYBIND11_CONCAT(pybind11_exec_, name)(PyObject *); \ + PYBIND11_PLUGIN_IMPL(name) { \ + PYBIND11_CHECK_PYTHON_VERSION \ + try { \ + pybind11::detail::ensure_internals(); \ + static ::pybind11::detail::slots_array mod_def_slots \ + = ::pybind11::detail::init_slots(&PYBIND11_CONCAT(pybind11_exec_, name), \ + ##__VA_ARGS__); \ + static PyModuleDef def{/* m_base */ PyModuleDef_HEAD_INIT, \ + /* m_name */ PYBIND11_TOSTRING(name), \ + /* m_doc */ nullptr, \ + /* m_size */ 0, \ + /* m_methods */ nullptr, \ + /* m_slots */ mod_def_slots.data(), \ + /* m_traverse */ nullptr, \ + /* m_clear */ nullptr, \ + /* m_free */ nullptr}; \ + return PyModuleDef_Init(&def); \ + } \ + PYBIND11_CATCH_INIT_EXCEPTIONS \ + return nullptr; \ + } + +#define PYBIND11_MODULE_EXEC(name, variable) \ + static void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ &); \ + int PYBIND11_CONCAT(pybind11_exec_, name)(PyObject * pm) { \ + try { \ + pybind11::detail::ensure_internals(); \ + auto m = pybind11::reinterpret_borrow<::pybind11::module_>(pm); \ + if (!pybind11::detail::get_cached_module(m.attr("__spec__").attr("name"))) { \ + PYBIND11_CONCAT(pybind11_init_, name)(m); \ + pybind11::detail::cache_completed_module(m); \ + } \ + return 0; \ + } \ + PYBIND11_CATCH_INIT_EXCEPTIONS \ + return -1; \ + } \ + void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ \ + & variable) // NOLINT(bugprone-macro-parentheses) + /** \rst This macro creates the entry point that will be invoked when the Python interpreter imports an extension module. The module name is given as the first argument and it should not be in quotes. The second macro argument defines a variable of type - `py::module_` which can be used to initialize the module. + ``py::module_`` which can be used to initialize the module. The entry point is marked as "maybe unused" to aid dead-code detection analysis: since the entry point is typically only looked up at runtime and not referenced @@ -454,24 +515,31 @@ PYBIND11_WARNING_POP return "Hello, World!"; }); } + + The third and subsequent macro arguments are optional (available since 2.13.0), and + can be used to mark the extension module as supporting various Python features. + + - ``mod_gil_not_used()`` + - ``multiple_interpreters::per_interpreter_gil()`` + - ``multiple_interpreters::shared_gil()`` + - ``multiple_interpreters::not_supported()`` + + .. code-block:: cpp + + PYBIND11_MODULE(example, m, py::mod_gil_not_used()) { + m.doc() = "pybind11 example module safe to run without the GIL"; + m.def("foo", []() { + return "Hello, Free-threaded World!"; + }); + } + \endrst */ -#define PYBIND11_MODULE(name, variable) \ - static ::pybind11::module_::module_def PYBIND11_CONCAT(pybind11_module_def_, name) \ - PYBIND11_MAYBE_UNUSED; \ - PYBIND11_MAYBE_UNUSED \ - static void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ &); \ - PYBIND11_PLUGIN_IMPL(name) { \ - PYBIND11_CHECK_PYTHON_VERSION \ - PYBIND11_ENSURE_INTERNALS_READY \ - auto m = ::pybind11::module_::create_extension_module( \ - PYBIND11_TOSTRING(name), nullptr, &PYBIND11_CONCAT(pybind11_module_def_, name)); \ - try { \ - PYBIND11_CONCAT(pybind11_init_, name)(m); \ - return m.ptr(); \ - } \ - PYBIND11_CATCH_INIT_EXCEPTIONS \ - } \ - void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ & (variable)) +#define PYBIND11_MODULE(name, variable, ...) \ + PYBIND11_MODULE_PYINIT(name, ##__VA_ARGS__) \ + PYBIND11_MODULE_EXEC(name, variable) + +// pop gnu-zero-variadic-macro-arguments +PYBIND11_WARNING_POP PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) @@ -527,7 +595,7 @@ enum class return_value_policy : uint8_t { object without taking ownership similar to the above return_value_policy::reference policy. In contrast to that policy, the function or property's implicit this argument (called the parent) is - considered to be the the owner of the return value (the child). + considered to be the owner of the return value (the child). pybind11 then couples the lifetime of the parent to the child via a reference relationship that ensures that the parent cannot be garbage collected while Python is still using the child. More advanced @@ -538,14 +606,10 @@ enum class return_value_policy : uint8_t { PYBIND11_NAMESPACE_BEGIN(detail) -inline static constexpr int log2(size_t n, int k = 0) { - return (n <= 1) ? k : log2(n >> 1, k + 1); -} +static constexpr int log2(size_t n, int k = 0) { return (n <= 1) ? k : log2(n >> 1, k + 1); } // Returns the size as a multiple of sizeof(void *), rounded up. -inline static constexpr size_t size_in_ptrs(size_t s) { - return 1 + ((s - 1) >> log2(sizeof(void *))); -} +static constexpr size_t size_in_ptrs(size_t s) { return 1 + ((s - 1) >> log2(sizeof(void *))); } /** * The space to allocate for simple layout instance holders (see below) in multiple of the size of @@ -610,6 +674,8 @@ struct instance { bool simple_instance_registered : 1; /// If true, get_internals().patients has an entry for this object bool has_patients : 1; + /// If true, this Python object needs to be kept alive for the lifetime of the C++ value. + bool is_alias : 1; /// Initializes all of the above type/values/holders data (but not the instance values /// themselves) @@ -632,6 +698,14 @@ struct instance { static_assert(std::is_standard_layout::value, "Internal error: `pybind11::detail::instance` is not standard layout!"); +// Some older compilers (e.g. gcc 9.4.0) require +// static_assert(always_false::value, "..."); +// instead of +// static_assert(false, "..."); +// to trigger the static_assert() in a template only if it is actually instantiated. +template +struct always_false : std::false_type {}; + /// from __cpp_future__ import (convenient aliases from C++14/17) #if defined(PYBIND11_CPP14) using std::conditional_t; @@ -649,7 +723,7 @@ template using remove_reference_t = typename std::remove_reference::type; #endif -#if defined(PYBIND11_CPP20) +#if defined(PYBIND11_CPP20) && defined(__cpp_lib_remove_cvref) using std::remove_cvref; using std::remove_cvref_t; #else @@ -661,6 +735,10 @@ template using remove_cvref_t = typename remove_cvref::type; #endif +/// Example usage: is_same_ignoring_cvref::value +template +using is_same_ignoring_cvref = std::is_same, U>; + /// Index sequences #if defined(PYBIND11_CPP14) using std::index_sequence; @@ -668,14 +746,49 @@ using std::make_index_sequence; #else template struct index_sequence {}; -template -struct make_index_sequence_impl : make_index_sequence_impl {}; -template -struct make_index_sequence_impl<0, S...> { +// Comments about the algorithm below. +// +// Credit: This is based on an algorithm by taocpp here: +// https://github.com/taocpp/sequences/blob/main/include/tao/seq/make_integer_sequence.hpp +// but significantly simplified. +// +// We build up a sequence S by repeatedly doubling its length and sometimes adding 1 to the end. +// E.g. if the current S is 0...3, then we either go to 0...7 or 0...8 on the next pass. +// The goal is to end with S = 0...N-1. +// The key insight is that the times we need to add an additional digit to S correspond +// exactly to the 1's in the binary representation of the number N. +// +// Invariants: +// - digit is a power of 2 +// - N_digit_is_1 is whether N's binary representation has a 1 in that digit's position. +// - end <= N +// - S is 0...end-1. +// - if digit > 0, end * digit * 2 <= N < (end+1) * digit * 2 +// +// The process starts with digit > N, end = 0, and S is empty. +// The process concludes with digit=0, in which case, end == N and S is 0...N-1. + +template // N_digit_is_1=false +struct make_index_sequence_impl + : make_index_sequence_impl { +}; +template +struct make_index_sequence_impl + : make_index_sequence_impl {}; +template +struct make_index_sequence_impl<0, false, N, end, S...> { using type = index_sequence; }; +constexpr size_t next_power_of_2(size_t N) { return N == 0 ? 1 : next_power_of_2(N >> 1) << 1; } template -using make_index_sequence = typename make_index_sequence_impl::type; +using make_index_sequence = + typename make_index_sequence_impl::type; #endif /// Make an index sequence of the indices of true arguments @@ -713,7 +826,7 @@ using void_t = typename void_t_impl::type; #endif /// Compile-time all/any/none of that check the boolean value of all template types -#if defined(__cpp_fold_expressions) && !(defined(_MSC_VER) && (_MSC_VER < 1916)) +#if defined(__cpp_fold_expressions) template using all_of = bool_constant<(Ts::value && ...)>; template @@ -754,7 +867,16 @@ template struct remove_class { using type = R(A...); }; - +#ifdef __cpp_noexcept_function_type +template +struct remove_class { + using type = R(A...); +}; +template +struct remove_class { + using type = R(A...); +}; +#endif /// Helper template to strip away type modifiers template struct intrinsic_type { @@ -903,8 +1025,7 @@ using is_template_base_of = decltype(is_template_base_of_impl::check((intrinsic_t *) nullptr)); #else struct is_template_base_of - : decltype(is_template_base_of_impl::check((intrinsic_t *) nullptr)) { -}; + : decltype(is_template_base_of_impl::check((intrinsic_t *) nullptr)){}; #endif /// Check if T is an instantiation of the template `Class`. For example: @@ -918,6 +1039,17 @@ struct is_instantiation> : std::true_type {}; template using is_shared_ptr = is_instantiation; +/// Detects whether static_cast(Base*) is valid, i.e. the inheritance is non-virtual. +/// Used to detect virtual bases: if this is false, pointer adjustments require the implicit_casts +/// chain rather than reinterpret_cast. +template +struct is_static_downcastable : std::false_type {}; +template +struct is_static_downcastable(std::declval()))>> + : std::true_type {}; + /// Check if T looks like an input iterator template struct is_input_iterator : std::false_type {}; @@ -940,14 +1072,30 @@ struct strip_function_object { using type = typename remove_class::type; }; +// Strip noexcept from a free function type (C++17: noexcept is part of the type). +template +struct remove_noexcept { + using type = T; +}; +#ifdef __cpp_noexcept_function_type +template +struct remove_noexcept { + using type = R(A...); +}; +#endif +template +using remove_noexcept_t = typename remove_noexcept::type; + // Extracts the function signature from a function, function pointer or lambda. +// Strips noexcept from the result so that factory/pickle_factory partial specializations +// (which match plain Return(Args...)) work correctly with noexcept callables (issue #2234). template > -using function_signature_t = conditional_t< +using function_signature_t = remove_noexcept_t::value, F, typename conditional_t::value || std::is_member_pointer::value, std::remove_pointer, - strip_function_object>::type>; + strip_function_object>::type>>; /// Returns true if the type looks like a lambda: that is, isn't a function, pointer or member /// pointer. Note that this can catch all sorts of other things, too; this is intended to be used @@ -1012,6 +1160,15 @@ PYBIND11_RUNTIME_EXCEPTION(reference_cast_error, PyExc_RuntimeError) /// Used in template struct format_descriptor {}; +template +struct format_descriptor< + T, + detail::enable_if_t::value>> { + static constexpr const char c = 'O'; + static constexpr const char value[2] = {c, '\0'}; + static std::string format() { return std::string(1, c); } +}; + PYBIND11_NAMESPACE_BEGIN(detail) // Returns the index of the given type in the type char array below, and in the list in numpy.h // The order here is: bool; 8 ints ((signed,unsigned)x(8,16,32,64)bits); float,double,long double; @@ -1087,6 +1244,36 @@ struct overload_cast_impl { -> decltype(pmf) { return pmf; } + + // Define const/non-const member-pointer selector pairs for qualifier combinations. + // The `qualifiers` parameter is used in type position, where extra parentheses are invalid. + // NOLINTBEGIN(bugprone-macro-parentheses) +#define PYBIND11_OVERLOAD_CAST_MEMBER_PTR(qualifiers) \ + template \ + constexpr auto operator()(Return (Class::*pmf)(Args...) qualifiers, std::false_type = {}) \ + const noexcept -> decltype(pmf) { \ + return pmf; \ + } \ + template \ + constexpr auto operator()(Return (Class::*pmf)(Args...) const qualifiers, std::true_type) \ + const noexcept -> decltype(pmf) { \ + return pmf; \ + } + PYBIND11_OVERLOAD_CAST_MEMBER_PTR(&) + PYBIND11_OVERLOAD_CAST_MEMBER_PTR(&&) + +#ifdef __cpp_noexcept_function_type + template + constexpr auto operator()(Return (*pf)(Args...) noexcept) const noexcept -> decltype(pf) { + return pf; + } + + PYBIND11_OVERLOAD_CAST_MEMBER_PTR(noexcept) + PYBIND11_OVERLOAD_CAST_MEMBER_PTR(& noexcept) + PYBIND11_OVERLOAD_CAST_MEMBER_PTR(&& noexcept) +#endif +#undef PYBIND11_OVERLOAD_CAST_MEMBER_PTR + // NOLINTEND(bugprone-macro-parentheses) }; PYBIND11_NAMESPACE_END(detail) @@ -1170,7 +1357,7 @@ try_get_shared_from_this(std::enable_shared_from_this *holder_value_ptr) { // Use at your own risk! // See also https://en.cppreference.com/w/cpp/memory/enable_shared_from_this, and in particular // the `std::shared_ptr gp1 = not_so_good.getptr();` and `try`-`catch` parts of the example. -#if defined(__cpp_lib_enable_shared_from_this) && (!defined(_MSC_VER) || _MSC_VER >= 1912) +#if defined(__cpp_lib_enable_shared_from_this) return holder_value_ptr->weak_from_this().lock(); #else try { @@ -1183,20 +1370,10 @@ try_get_shared_from_this(std::enable_shared_from_this *holder_value_ptr) { // For silencing "unused" compiler warnings in special situations. template -#if defined(_MSC_VER) && _MSC_VER < 1920 // MSVC 2017 -constexpr -#endif - inline void - silence_unused_warnings(Args &&...) { -} +inline void silence_unused_warnings(Args &&...) {} -// MSVC warning C4100: Unreferenced formal parameter -#if defined(_MSC_VER) && _MSC_VER <= 1916 -# define PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(...) \ - detail::silence_unused_warnings(__VA_ARGS__) -#else -# define PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(...) -#endif +// MSVC warning C4100: Unreferenced formal parameter (only incorrect before MSVC 2019) +#define PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(...) // GCC -Wunused-but-set-parameter All GCC versions (as of July 2021). #if defined(__GNUG__) && !defined(__clang__) && !defined(__INTEL_COMPILER) @@ -1224,12 +1401,18 @@ constexpr // std::move(args) #endif -// Pybind offers detailed error messages by default for all builts that are debug (through the -// negation of ndebug). This can also be manually enabled by users, for any builds, through -// defining PYBIND11_DETAILED_ERROR_MESSAGES. +// Pybind offers detailed error messages by default in debug builds (through the negation of +// NDEBUG). This can also be manually enabled by users for any build by defining +// PYBIND11_DETAILED_ERROR_MESSAGES. This information is primarily useful for those +// who are writing (as opposed to merely using) libraries that use pybind11. #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) && !defined(NDEBUG) # define PYBIND11_DETAILED_ERROR_MESSAGES #endif +// CPython 3.11+ provides Py_TPFLAGS_MANAGED_DICT, but PyPy3.11 does not, see PR #5508. +#if PY_VERSION_HEX < 0x030B0000 || defined(PYPY_VERSION) +# define PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET +#endif + PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/detail/cpp_conduit.h b/lib/pybind11/include/pybind11/detail/cpp_conduit.h new file mode 100644 index 0000000..49c199e --- /dev/null +++ b/lib/pybind11/include/pybind11/detail/cpp_conduit.h @@ -0,0 +1,75 @@ +// Copyright (c) 2024 The pybind Community. + +#pragma once + +#include + +#include "common.h" +#include "internals.h" + +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +// Forward declaration needed here: Refactoring opportunity. +extern "C" inline PyObject *pybind11_object_new(PyTypeObject *type, PyObject *, PyObject *); + +inline bool type_is_managed_by_our_internals(PyTypeObject *type_obj) { +#if defined(PYPY_VERSION) + auto &internals = get_internals(); + return bool(internals.registered_types_py.find(type_obj) + != internals.registered_types_py.end()); +#else + return (type_obj->tp_new == pybind11_object_new); +#endif +} + +inline bool is_instance_method_of_type(PyTypeObject *type_obj, PyObject *attr_name) { + PyObject *descr = _PyType_Lookup(type_obj, attr_name); + return ((descr != nullptr) && PyInstanceMethod_Check(descr)); +} + +inline object try_get_cpp_conduit_method(PyObject *obj) { + if (PyType_Check(obj)) { + return object(); + } + PyTypeObject *type_obj = Py_TYPE(obj); + str attr_name("_pybind11_conduit_v1_"); + bool assumed_to_be_callable = false; + if (type_is_managed_by_our_internals(type_obj)) { + if (!is_instance_method_of_type(type_obj, attr_name.ptr())) { + return object(); + } + assumed_to_be_callable = true; + } + PyObject *method = PyObject_GetAttr(obj, attr_name.ptr()); + if (method == nullptr) { + PyErr_Clear(); + return object(); + } + if (!assumed_to_be_callable && PyCallable_Check(method) == 0) { + Py_DECREF(method); + return object(); + } + return reinterpret_steal(method); +} + +inline void *try_raw_pointer_ephemeral_from_cpp_conduit(handle src, + const std::type_info *cpp_type_info) { + object method = try_get_cpp_conduit_method(src.ptr()); + if (method) { + capsule cpp_type_info_capsule(const_cast(static_cast(cpp_type_info)), + typeid(std::type_info).name()); + object cpp_conduit = method(bytes(PYBIND11_PLATFORM_ABI_ID), + cpp_type_info_capsule, + bytes("raw_pointer_ephemeral")); + if (isinstance(cpp_conduit)) { + return reinterpret_borrow(cpp_conduit).get_pointer(); + } + } + return nullptr; +} + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/detail/descr.h b/lib/pybind11/include/pybind11/detail/descr.h index e7a5e2c..4f5ae34 100644 --- a/lib/pybind11/include/pybind11/detail/descr.h +++ b/lib/pybind11/include/pybind11/detail/descr.h @@ -99,6 +99,26 @@ constexpr descr<1, Type> const_name() { return {'%'}; } +// Use a different name based on whether the parameter is used as input or output +template +constexpr descr io_name(char const (&text1)[N1], char const (&text2)[N2]) { + return const_name("@") + const_name(text1) + const_name("@") + const_name(text2) + + const_name("@"); +} + +// Ternary description for io_name (like the numeric type_caster) +template +constexpr enable_if_t> +io_name(char const (&text1)[N1], char const (&text2)[N2], char const (&)[N3], char const (&)[N4]) { + return io_name(text1, text2); +} + +template +constexpr enable_if_t> +io_name(char const (&)[N1], char const (&)[N2], char const (&text3)[N3], char const (&text4)[N4]) { + return io_name(text3, text4); +} + // If "_" is defined as a macro, py::detail::_ cannot be provided. // It is therefore best to use py::detail::const_name universally. // This block is for backward compatibility only. @@ -137,22 +157,75 @@ constexpr descr<1, Type> _() { #endif // #ifndef _ constexpr descr<0> concat() { return {}; } +constexpr descr<0> union_concat() { return {}; } template constexpr descr concat(const descr &descr) { return descr; } +template +constexpr descr union_concat(const descr &descr) { + return descr; +} + +template +constexpr descr operator|(const descr &a, + const descr &b) { + return a + const_name(" | ") + b; +} + +#ifdef __cpp_fold_expressions +template +constexpr descr operator,(const descr &a, + const descr &b) { + return a + const_name(", ") + b; +} + +template +constexpr auto concat(const descr &d, const Args &...args) { + return (d, ..., args); +} + +template +constexpr auto union_concat(const descr &d, const Args &...args) { + return (d | ... | args); +} + +#else template constexpr auto concat(const descr &d, const Args &...args) -> decltype(std::declval>() + concat(args...)) { return d + const_name(", ") + concat(args...); } +template +constexpr auto union_concat(const descr &d, const Args &...args) + -> decltype(std::declval>() + union_concat(args...)) { + return d + const_name(" | ") + union_concat(args...); +} + +#endif + template constexpr descr type_descr(const descr &descr) { return const_name("{") + descr + const_name("}"); } +template +constexpr descr arg_descr(const descr &descr) { + return const_name("@^") + descr + const_name("@!"); +} + +template +constexpr descr return_descr(const descr &descr) { + return const_name("@$") + descr + const_name("@!"); +} + +template +constexpr descr inv_descr(const descr &descr) { + return const_name("@~") + descr + const_name("@!"); +} + PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h b/lib/pybind11/include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h new file mode 100644 index 0000000..7c00fe9 --- /dev/null +++ b/lib/pybind11/include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h @@ -0,0 +1,39 @@ +// Copyright (c) 2021 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#pragma once + +#include "common.h" + +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +template +struct dynamic_raw_ptr_cast_is_possible : std::false_type {}; + +template +struct dynamic_raw_ptr_cast_is_possible< + To, + From, + detail::enable_if_t::value && std::is_polymorphic::value>> + : std::true_type {}; + +template ::value, int> = 0> +To *dynamic_raw_ptr_cast_if_possible(From * /*ptr*/) { + return nullptr; +} + +template ::value, int> = 0> +To *dynamic_raw_ptr_cast_if_possible(From *ptr) { + return dynamic_cast(ptr); +} + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/detail/exception_translation.h b/lib/pybind11/include/pybind11/detail/exception_translation.h new file mode 100644 index 0000000..22ae8a1 --- /dev/null +++ b/lib/pybind11/include/pybind11/detail/exception_translation.h @@ -0,0 +1,71 @@ +/* + pybind11/detail/exception_translation.h: means to translate C++ exceptions to Python exceptions + + Copyright (c) 2024 The Pybind Development Team. + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "common.h" +#include "internals.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +// Apply all the extensions translators from a list +// Return true if one of the translators completed without raising an exception +// itself. Return of false indicates that if there are other translators +// available, they should be tried. +inline bool apply_exception_translators(std::forward_list &translators) { + auto last_exception = std::current_exception(); + + for (auto &translator : translators) { + try { + translator(last_exception); + return true; + } catch (...) { + last_exception = std::current_exception(); + } + } + return false; +} + +inline void try_translate_exceptions() { + /* When an exception is caught, give each registered exception + translator a chance to translate it to a Python exception. First + all module-local translators will be tried in reverse order of + registration. If none of the module-locale translators handle + the exception (or there are no module-locale translators) then + the global translators will be tried, also in reverse order of + registration. + + A translator may choose to do one of the following: + + - catch the exception and call py::set_error() + to set a standard (or custom) Python exception, or + - do nothing and let the exception fall through to the next translator, or + - delegate translation to the next translator by throwing a new type of exception. + */ + + bool handled = with_exception_translators( + [&](std::forward_list &exception_translators, + std::forward_list &local_exception_translators) { + if (detail::apply_exception_translators(local_exception_translators)) { + return true; + } + if (detail::apply_exception_translators(exception_translators)) { + return true; + } + return false; + }); + + if (!handled) { + set_error(PyExc_SystemError, "Exception escaped from default exception translator!"); + } +} + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/detail/function_record_pyobject.h b/lib/pybind11/include/pybind11/detail/function_record_pyobject.h new file mode 100644 index 0000000..42b0e96 --- /dev/null +++ b/lib/pybind11/include/pybind11/detail/function_record_pyobject.h @@ -0,0 +1,191 @@ +// Copyright (c) 2024-2025 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// For background see the description of PR google/pybind11clif#30099. + +#pragma once + +#include +#include +#include + +#include "common.h" + +#include +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +struct function_record_PyObject { + PyObject_HEAD + function_record *cpp_func_rec; +}; + +PYBIND11_NAMESPACE_BEGIN(function_record_PyTypeObject_methods) + +PyObject *tp_new_impl(PyTypeObject *type, PyObject *args, PyObject *kwds); +PyObject *tp_alloc_impl(PyTypeObject *type, Py_ssize_t nitems); +int tp_init_impl(PyObject *self, PyObject *args, PyObject *kwds); +void tp_dealloc_impl(PyObject *self); +void tp_free_impl(void *self); + +static PyObject *reduce_ex_impl(PyObject *self, PyObject *, PyObject *); + +static PyMethodDef tp_methods_impl[] + = {{"__reduce_ex__", + // reduce_ex_impl is a PyCFunctionWithKeywords, but PyMethodDef + // requires a PyCFunction. The cast through void* is safe and + // idiomatic with METH_KEYWORDS, and it successfully sidesteps + // unhelpful compiler warnings. + // NOLINTNEXTLINE(bugprone-casting-through-void) + reinterpret_cast(reinterpret_cast(reduce_ex_impl)), + METH_VARARGS | METH_KEYWORDS, + nullptr}, + {nullptr, nullptr, 0, nullptr}}; + +// Python 3.12+ emits a DeprecationWarning for heap types whose tp_name does +// not contain a dot ('.') and that lack a __module__ attribute. For pybind11's +// internal function_record type, we do not have an actual module object to +// attach, so we cannot use PyType_FromModuleAndSpec to set __module__ +// automatically. +// +// As a workaround, we define a "qualified" type name that includes a dummy +// module name (PYBIND11_DUMMY_MODULE_NAME). This is non‑idiomatic but avoids +// the deprecation warning, and results in reprs like +// +// +// +// even though no real pybind11_builtins module exists. If pybind11 gains an +// actual module object in the future, this code should switch to +// PyType_FromModuleAndSpec and drop the dummy module workaround. +// +// Note that this name is versioned. +#define PYBIND11_DETAIL_FUNCTION_RECORD_TP_PLAINNAME \ + "pybind11_detail_function_record_" PYBIND11_DETAIL_FUNCTION_RECORD_ABI_ID \ + "_" PYBIND11_PLATFORM_ABI_ID +constexpr char tp_plainname_impl[] = PYBIND11_DETAIL_FUNCTION_RECORD_TP_PLAINNAME; +constexpr char tp_qualname_impl[] + = PYBIND11_DUMMY_MODULE_NAME "." PYBIND11_DETAIL_FUNCTION_RECORD_TP_PLAINNAME; + +PYBIND11_NAMESPACE_END(function_record_PyTypeObject_methods) + +static PyType_Slot function_record_PyType_Slots[] = { + {Py_tp_dealloc, + reinterpret_cast(function_record_PyTypeObject_methods::tp_dealloc_impl)}, + {Py_tp_methods, + reinterpret_cast(function_record_PyTypeObject_methods::tp_methods_impl)}, + {Py_tp_init, reinterpret_cast(function_record_PyTypeObject_methods::tp_init_impl)}, + {Py_tp_alloc, reinterpret_cast(function_record_PyTypeObject_methods::tp_alloc_impl)}, + {Py_tp_new, reinterpret_cast(function_record_PyTypeObject_methods::tp_new_impl)}, + {Py_tp_free, reinterpret_cast(function_record_PyTypeObject_methods::tp_free_impl)}, + {0, nullptr}}; + +static PyType_Spec function_record_PyType_Spec + = {function_record_PyTypeObject_methods::tp_qualname_impl, + sizeof(function_record_PyObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE, + function_record_PyType_Slots}; + +inline PyTypeObject *get_function_record_PyTypeObject() { + PYBIND11_LOCK_INTERNALS(get_internals()); + PyTypeObject *&py_type_obj = detail::get_local_internals().function_record_py_type; + if (!py_type_obj) { + PyObject *py_obj = PyType_FromSpec(&function_record_PyType_Spec); + if (py_obj == nullptr) { + throw error_already_set(); + } + py_type_obj = reinterpret_cast(py_obj); + } + return py_type_obj; +} + +inline bool is_function_record_PyObject(PyObject *obj) { + if (PyType_Check(obj) != 0) { + return false; + } + PyTypeObject *obj_type = Py_TYPE(obj); + + PyTypeObject *frtype = get_function_record_PyTypeObject(); + + // Fast path (pointer comparison). + if (obj_type == frtype) { + return true; + } + // This works across extension modules. Note that tp_name is versioned. + if (strcmp(obj_type->tp_name, function_record_PyTypeObject_methods::tp_qualname_impl) == 0 + || strcmp(obj_type->tp_name, function_record_PyTypeObject_methods::tp_plainname_impl) + == 0) { + return true; + } + return false; +} + +inline function_record *function_record_ptr_from_PyObject(PyObject *obj) { + if (is_function_record_PyObject(obj)) { + return (reinterpret_cast(obj))->cpp_func_rec; + } + return nullptr; +} + +inline object function_record_PyObject_New() { + auto *py_func_rec = PyObject_New(function_record_PyObject, get_function_record_PyTypeObject()); + if (py_func_rec == nullptr) { + throw error_already_set(); + } + py_func_rec->cpp_func_rec = nullptr; // For clarity/purity. Redundant in practice. + return reinterpret_steal(reinterpret_cast(py_func_rec)); +} + +PYBIND11_NAMESPACE_BEGIN(function_record_PyTypeObject_methods) + +// Guard against accidents & oversights, in particular when porting to future Python versions. +inline PyObject *tp_new_impl(PyTypeObject *, PyObject *, PyObject *) { + pybind11_fail("UNEXPECTED CALL OF function_record_PyTypeObject_methods::tp_new_impl"); + // return nullptr; // Unreachable. +} + +inline PyObject *tp_alloc_impl(PyTypeObject *, Py_ssize_t) { + pybind11_fail("UNEXPECTED CALL OF function_record_PyTypeObject_methods::tp_alloc_impl"); + // return nullptr; // Unreachable. +} + +inline int tp_init_impl(PyObject *, PyObject *, PyObject *) { + pybind11_fail("UNEXPECTED CALL OF function_record_PyTypeObject_methods::tp_init_impl"); + // return -1; // Unreachable. +} + +inline void tp_free_impl(void *) { + pybind11_fail("UNEXPECTED CALL OF function_record_PyTypeObject_methods::tp_free_impl"); +} + +inline PyObject *reduce_ex_impl(PyObject *self, PyObject *, PyObject *) { + // Deliberately ignoring the arguments for simplicity (expected is `protocol: int`). + const function_record *rec = function_record_ptr_from_PyObject(self); + if (rec == nullptr) { + pybind11_fail( + "FATAL: function_record_PyTypeObject reduce_ex_impl(): cannot obtain cpp_func_rec."); + } + if (rec->name != nullptr && rec->name[0] != '\0' && rec->scope + && PyModule_Check(rec->scope.ptr()) != 0) { + object scope_module = get_scope_module(rec->scope); + if (scope_module) { + auto builtins = reinterpret_borrow(PyEval_GetBuiltins()); + auto builtins_eval = builtins["eval"]; + auto reconstruct_args = make_tuple(str("__import__('importlib').import_module('") + + scope_module + str("')")); + return make_tuple(std::move(builtins_eval), std::move(reconstruct_args)) + .release() + .ptr(); + } + } + set_error(PyExc_RuntimeError, repr(self) + str(" is not pickleable.")); + return nullptr; +} + +PYBIND11_NAMESPACE_END(function_record_PyTypeObject_methods) + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/detail/function_ref.h b/lib/pybind11/include/pybind11/detail/function_ref.h new file mode 100644 index 0000000..a81bdfe --- /dev/null +++ b/lib/pybind11/include/pybind11/detail/function_ref.h @@ -0,0 +1,101 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +//===- llvm/ADT/STLFunctionalExtras.h - Extras for -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains a header-only class template that provides functionality +// similar to std::function but with non-owning semantics. It is a template-only +// implementation that requires no additional library linking. +// +//===----------------------------------------------------------------------===// + +/// An efficient, type-erasing, non-owning reference to a callable. This is +/// intended for use as the type of a function parameter that is not used +/// after the function in question returns. +/// +/// This class does not own the callable, so it is not in general safe to store +/// a FunctionRef. + +// pybind11: modified again from executorch::runtime::FunctionRef +// - renamed back to function_ref +// - use pybind11 enable_if_t, remove_cvref_t, and remove_reference_t +// - lint suppressions + +// torch::executor: modified from llvm::function_ref +// - renamed to FunctionRef +// - removed LLVM_GSL_POINTER and LLVM_LIFETIME_BOUND macro uses +// - use namespaced internal::remove_cvref_t + +#pragma once + +#include + +#include +#include +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +//===----------------------------------------------------------------------===// +// Features from C++20 +//===----------------------------------------------------------------------===// + +template +class function_ref; + +template +class function_ref { + Ret (*callback)(intptr_t callable, Params... params) = nullptr; + intptr_t callable; + + template + // NOLINTNEXTLINE(performance-unnecessary-value-param) + static Ret callback_fn(intptr_t callable, Params... params) { + // NOLINTNEXTLINE(performance-no-int-to-ptr) + return (*reinterpret_cast(callable))(std::forward(params)...); + } + +public: + function_ref() = default; + // NOLINTNEXTLINE(google-explicit-constructor) + function_ref(std::nullptr_t) {} + + template + // NOLINTNEXTLINE(google-explicit-constructor) + function_ref( + Callable &&callable, + // This is not the copy-constructor. + enable_if_t, function_ref>::value> * = nullptr, + // Functor must be callable and return a suitable type. + enable_if_t< + std::is_void::value + || std::is_convertible()(std::declval()...)), + Ret>::value> * = nullptr) + : callback(callback_fn>), + callable(reinterpret_cast(&callable)) {} + + // NOLINTNEXTLINE(performance-unnecessary-value-param) + Ret operator()(Params... params) const { + return callback(callable, std::forward(params)...); + } + + explicit operator bool() const { return callback; } + + bool operator==(const function_ref &Other) const { + return callable == Other.callable; + } +}; +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/detail/holder_caster_foreign_helpers.h b/lib/pybind11/include/pybind11/detail/holder_caster_foreign_helpers.h new file mode 100644 index 0000000..cae571b --- /dev/null +++ b/lib/pybind11/include/pybind11/detail/holder_caster_foreign_helpers.h @@ -0,0 +1,104 @@ +/* + pybind11/detail/holder_caster_foreign_helpers.h: Logic to implement + set_foreign_holder() in copyable_ and movable_holder_caster. + + Copyright (c) 2025 Hudson River Trading LLC + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include + +#include "common.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +struct holder_caster_foreign_helpers { + struct py_deleter { + void operator()(const void *) const noexcept { + // Don't run the deleter if the interpreter has been shut down + if (Py_IsInitialized() == 0) { + return; + } + gil_scoped_acquire guard; + Py_DECREF(o); + } + + PyObject *o; + }; + + // Downcast shared_ptr from the enable_shared_from_this base to the target type. + // SFINAE probe: use static_pointer_cast when the static downcast is valid (common case), + // fall back to dynamic_pointer_cast when it isn't (virtual inheritance — issue #5989). + // We can't use dynamic_pointer_cast unconditionally because it requires polymorphic types; + // we can't use is_polymorphic to choose because that's orthogonal to virtual inheritance. + // (The implementation uses the "tag dispatch via overload priority" trick.) + template + static auto esft_downcast(const std::shared_ptr &existing, int /*preferred*/) + -> decltype(static_cast(std::declval()), std::shared_ptr()) { + return std::static_pointer_cast(existing); + } + + template + static std::shared_ptr esft_downcast(const std::shared_ptr &existing, + ... /*fallback*/) { + return std::dynamic_pointer_cast(existing); + } + + template + static auto set_via_shared_from_this(type *value, std::shared_ptr *holder_out) + -> decltype(value->shared_from_this(), bool()) { + // object derives from enable_shared_from_this; + // try to reuse an existing shared_ptr if one is known + if (auto existing = try_get_shared_from_this(value)) { + *holder_out = esft_downcast(existing, 0); + return true; + } + return false; + } + + template + static bool set_via_shared_from_this(void *, std::shared_ptr *) { + return false; + } + + template + static bool set_foreign_holder(handle src, type *value, std::shared_ptr *holder_out) { + // We only support using std::shared_ptr for foreign T, and + // it's done by creating a new shared_ptr control block that + // owns a reference to the original Python object. + if (value == nullptr) { + *holder_out = {}; + return true; + } + if (set_via_shared_from_this(value, holder_out)) { + return true; + } + *holder_out = std::shared_ptr(value, py_deleter{src.inc_ref().ptr()}); + return true; + } + + template + static bool + set_foreign_holder(handle src, const type *value, std::shared_ptr *holder_out) { + std::shared_ptr holder_mut; + if (set_foreign_holder(src, const_cast(value), &holder_mut)) { + *holder_out = holder_mut; + return true; + } + return false; + } + + template + static bool set_foreign_holder(handle, type *, ...) { + throw cast_error("Unable to cast foreign type to held instance -- " + "only std::shared_ptr is supported in this case"); + } +}; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/detail/init.h b/lib/pybind11/include/pybind11/detail/init.h index 9f71278..a1083f8 100644 --- a/lib/pybind11/include/pybind11/detail/init.h +++ b/lib/pybind11/include/pybind11/detail/init.h @@ -10,6 +10,7 @@ #pragma once #include "class.h" +#include "using_smart_holder.h" PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) @@ -36,7 +37,7 @@ class type_caster { PYBIND11_NAMESPACE_BEGIN(initimpl) -inline void no_nullptr(void *ptr) { +inline void no_nullptr(const void *ptr) { if (!ptr) { throw type_error("pybind11::init(): factory function returned nullptr"); } @@ -60,12 +61,12 @@ bool is_alias(Cpp *ptr) { } // Failing fallback version of the above for a no-alias class (always returns false) template -constexpr bool is_alias(void *) { +constexpr bool is_alias(const void *) { return false; } // Constructs and returns a new object; if the given arguments don't map to a constructor, we fall -// back to brace aggregate initiailization so that for aggregate initialization can be used with +// back to brace aggregate initialization so that for aggregate initialization can be used with // py::init, e.g. `py::init` to initialize a `struct T { int a; int b; }`. For // non-aggregate types, we need to use an ordinary T(...) constructor (invoking as `T{...}` usually // works, but will not do the expected thing when `T` has an `initializer_list` constructor). @@ -128,11 +129,13 @@ void construct(value_and_holder &v_h, Cpp *ptr, bool need_alias) { // the holder and destruction happens when we leave the C++ scope, and the holder // class gets to handle the destruction however it likes. v_h.value_ptr() = ptr; - v_h.set_instance_registered(true); // To prevent init_instance from registering it - v_h.type->init_instance(v_h.inst, nullptr); // Set up the holder + v_h.set_instance_registered(true); // Trick to prevent init_instance from registering it + // DANGER ZONE BEGIN: exceptions will leave v_h in an invalid state. + v_h.type->init_instance(v_h.inst, nullptr); // Set up the holder Holder temp_holder(std::move(v_h.holder>())); // Steal the holder v_h.type->dealloc(v_h); // Destroys the moved-out holder remains, resets value ptr to null v_h.set_instance_registered(false); + // DANGER ZONE END. construct_alias_from_cpp(is_alias_constructible{}, v_h, std::move(*ptr)); } else { @@ -153,7 +156,7 @@ void construct(value_and_holder &v_h, Alias *alias_ptr, bool) { // holder. This also handles types like std::shared_ptr and std::unique_ptr where T is a // derived type (through those holder's implicit conversion from derived class holder // constructors). -template +template >::value, int> = 0> void construct(value_and_holder &v_h, Holder holder, bool need_alias) { PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); auto *ptr = holder_helper>::get(holder); @@ -164,7 +167,12 @@ void construct(value_and_holder &v_h, Holder holder, bool need_alias) { "is not an alias instance"); } - v_h.value_ptr() = ptr; + // Cast away constness to store in void* storage. + // The value_and_holder storage is fundamentally untyped (void**), so we lose + // const-correctness here by design. The const qualifier will be restored + // when the pointer is later retrieved and cast back to the original type. + // This explicit const_cast makes the const-removal clearly visible. + v_h.value_ptr() = const_cast(static_cast(ptr)); v_h.type->init_instance(v_h.inst, &holder); } @@ -175,7 +183,7 @@ void construct(value_and_holder &v_h, Holder holder, bool need_alias) { template void construct(value_and_holder &v_h, Cpp &&result, bool need_alias) { PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); - static_assert(std::is_move_constructible>::value, + static_assert(is_move_constructible>::value, "pybind11::init() return-by-value factory function requires a movable class"); if (Class::has_alias && need_alias) { construct_alias_from_cpp(is_alias_constructible{}, v_h, std::move(result)); @@ -190,11 +198,97 @@ void construct(value_and_holder &v_h, Cpp &&result, bool need_alias) { template void construct(value_and_holder &v_h, Alias &&result, bool) { static_assert( - std::is_move_constructible>::value, + is_move_constructible>::value, "pybind11::init() return-by-alias-value factory function requires a movable alias class"); v_h.value_ptr() = new Alias(std::move(result)); } +template +smart_holder init_smart_holder_from_unique_ptr(std::unique_ptr &&unq_ptr, + bool void_cast_raw_ptr) { + void *void_ptr = void_cast_raw_ptr ? static_cast(unq_ptr.get()) : nullptr; + return smart_holder::from_unique_ptr(std::move(unq_ptr), void_ptr); +} + +template >, + detail::enable_if_t>::value, int> = 0> +void construct(value_and_holder &v_h, std::unique_ptr, D> &&unq_ptr, bool need_alias) { + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); + auto *ptr = unq_ptr.get(); + no_nullptr(ptr); + if (Class::has_alias && need_alias && !is_alias(ptr)) { + throw type_error("pybind11::init(): construction failed: returned std::unique_ptr pointee " + "is not an alias instance"); + } + // Here and below: if the new object is a trampoline, the shared_from_this mechanism needs + // to be prevented from accessing the smart_holder vptr, because it does not keep the + // trampoline Python object alive. For types that don't inherit from enable_shared_from_this + // it does not matter if void_cast_raw_ptr is true or false, therefore it's not necessary + // to also inspect the type. + auto smhldr = init_smart_holder_from_unique_ptr( + std::move(unq_ptr), /*void_cast_raw_ptr*/ Class::has_alias && is_alias(ptr)); + v_h.value_ptr() = ptr; + v_h.type->init_instance(v_h.inst, &smhldr); +} + +template >, + detail::enable_if_t>::value, int> = 0> +void construct(value_and_holder &v_h, + std::unique_ptr, D> &&unq_ptr, + bool /*need_alias*/) { + auto *ptr = unq_ptr.get(); + no_nullptr(ptr); + auto smhldr + = init_smart_holder_from_unique_ptr(std::move(unq_ptr), /*void_cast_raw_ptr*/ true); + v_h.value_ptr() = ptr; + v_h.type->init_instance(v_h.inst, &smhldr); +} + +template +void construct_from_shared_ptr(value_and_holder &v_h, + std::shared_ptr &&shd_ptr, + bool need_alias) { + static_assert(std::is_same>::value + || std::is_same>::value, + "Expected (const) Cpp as shared_ptr pointee"); + auto *ptr = shd_ptr.get(); + no_nullptr(ptr); + if (Class::has_alias && need_alias && !is_alias(ptr)) { + throw type_error("pybind11::init(): construction failed: returned std::shared_ptr pointee " + "is not an alias instance"); + } + // Cast to non-const if needed, consistent with internal design + auto smhldr + = smart_holder::from_shared_ptr(std::const_pointer_cast>(std::move(shd_ptr))); + v_h.value_ptr() = const_cast *>(ptr); + v_h.type->init_instance(v_h.inst, &smhldr); +} + +template >::value, int> = 0> +void construct(value_and_holder &v_h, std::shared_ptr> &&shd_ptr, bool need_alias) { + construct_from_shared_ptr, Class>(v_h, std::move(shd_ptr), need_alias); +} + +template >::value, int> = 0> +void construct(value_and_holder &v_h, + std::shared_ptr> &&shd_ptr, + bool need_alias) { + construct_from_shared_ptr, Class>(v_h, std::move(shd_ptr), need_alias); +} + +template >::value, int> = 0> +void construct(value_and_holder &v_h, + std::shared_ptr> &&shd_ptr, + bool /*need_alias*/) { + auto *ptr = shd_ptr.get(); + no_nullptr(ptr); + auto smhldr = smart_holder::from_shared_ptr(shd_ptr); + v_h.value_ptr() = ptr; + v_h.type->init_instance(v_h.inst, &smhldr); +} + // Implementing class for py::init<...>() template struct constructor { @@ -202,18 +296,18 @@ struct constructor { static void execute(Class &cl, const Extra &...extra) { cl.def( "__init__", - [](value_and_holder &v_h, Args... args) { + [](value_and_holder &v_h, + Args... args) { // NOLINT(performance-unnecessary-value-param) v_h.value_ptr() = construct_or_initialize>(std::forward(args)...); }, is_new_style_constructor(), extra...); } - template < - typename Class, - typename... Extra, - enable_if_t, Args...>::value, int> - = 0> + template , Args...>::value, + int> = 0> static void execute(Class &cl, const Extra &...extra) { cl.def( "__init__", @@ -230,11 +324,10 @@ struct constructor { extra...); } - template < - typename Class, - typename... Extra, - enable_if_t, Args...>::value, int> - = 0> + template , Args...>::value, + int> = 0> static void execute(Class &cl, const Extra &...extra) { cl.def( "__init__", @@ -250,11 +343,10 @@ struct constructor { // Implementing class for py::init_alias<...>() template struct alias_constructor { - template < - typename Class, - typename... Extra, - enable_if_t, Args...>::value, int> - = 0> + template , Args...>::value, + int> = 0> static void execute(Class &cl, const Extra &...extra) { cl.def( "__init__", @@ -275,8 +367,15 @@ template ` = `void_type()`, which the dual-factory +// specialization can also decompose as `AReturn(AArgs...)` with `AReturn=void_type` +// and `AArgs={}`. template -struct factory { +struct factory { remove_reference_t class_factory; // NOLINTNEXTLINE(google-explicit-constructor) @@ -380,7 +479,16 @@ void setstate(value_and_holder &v_h, std::pair &&result, bool need_alias) // See PR #2972 for details. return; } - setattr((PyObject *) v_h.inst, "__dict__", d); + // Our tests never run into an unset dict, but being careful here for now (see #5658) + auto dict = getattr(reinterpret_cast(v_h.inst), "__dict__", none()); + if (dict.is_none()) { + setattr(reinterpret_cast(v_h.inst), "__dict__", d); + } else { + // Keep the original object dict and just update it + if (PyDict_Update(dict.ptr(), d.ptr()) < 0) { + throw error_already_set(); + } + } } /// Implementation for py::pickle(GetState, SetState) @@ -397,9 +505,15 @@ template struct pickle_factory { - static_assert(std::is_same, intrinsic_t>::value, - "The type returned by `__getstate__` must be the same " - "as the argument accepted by `__setstate__`"); + using Ret = intrinsic_t; + using Arg = intrinsic_t; + + // Subclasses are now allowed for support between type hint and generic versions of types + // (e.g.) typing::List <--> list + static_assert(std::is_same::value || std::is_base_of::value + || std::is_base_of::value, + "The type returned by `__getstate__` must be the same or subclass of the " + "argument accepted by `__setstate__`"); remove_reference_t get; remove_reference_t set; @@ -408,7 +522,7 @@ struct pickle_factory { template void execute(Class &cl, const Extra &...extra) && { - cl.def("__getstate__", std::move(get)); + cl.def("__getstate__", std::move(get), pos_only()); #if defined(PYBIND11_CPP14) cl.def( diff --git a/lib/pybind11/include/pybind11/detail/internals.h b/lib/pybind11/include/pybind11/detail/internals.h index 05e36ad..7107b70 100644 --- a/lib/pybind11/include/pybind11/detail/internals.h +++ b/lib/pybind11/include/pybind11/detail/internals.h @@ -9,15 +9,20 @@ #pragma once -#include "common.h" - -#if defined(WITH_THREAD) && defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) -# include "../gil.h" -#endif +#include +#include +#include +#include -#include "../pytypes.h" +#include "common.h" +#include "struct_smart_holder.h" +#include +#include #include +#include +#include +#include /// Tracks the `internals` and `type_info` ABI version independent of the main library version. /// @@ -34,79 +39,139 @@ /// further ABI-incompatible changes may be made before the ABI is officially /// changed to the new version. #ifndef PYBIND11_INTERNALS_VERSION -# define PYBIND11_INTERNALS_VERSION 4 +# define PYBIND11_INTERNALS_VERSION 12 +#endif + +#if PYBIND11_INTERNALS_VERSION < 12 +# error "PYBIND11_INTERNALS_VERSION 12 is the minimum for all platforms for pybind11 v3.1.0" #endif PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) using ExceptionTranslator = void (*)(std::exception_ptr); +// The old Python Thread Local Storage (TLS) API is deprecated in Python 3.7 in favor of the new +// Thread Specific Storage (TSS) API. +// Avoid unnecessary allocation of `Py_tss_t`, since we cannot use +// `Py_LIMITED_API` anyway. +#define PYBIND11_TLS_KEY_REF Py_tss_t & +#if defined(__clang__) +# define PYBIND11_TLS_KEY_INIT(var) \ + _Pragma("clang diagnostic push") /**/ \ + _Pragma("clang diagnostic ignored \"-Wmissing-field-initializers\"") /**/ \ + Py_tss_t var = Py_tss_NEEDS_INIT; \ + _Pragma("clang diagnostic pop") +#elif defined(__GNUC__) && !defined(__INTEL_COMPILER) +# define PYBIND11_TLS_KEY_INIT(var) \ + _Pragma("GCC diagnostic push") /**/ \ + _Pragma("GCC diagnostic ignored \"-Wmissing-field-initializers\"") /**/ \ + Py_tss_t var = Py_tss_NEEDS_INIT; \ + _Pragma("GCC diagnostic pop") +#else +# define PYBIND11_TLS_KEY_INIT(var) Py_tss_t var = Py_tss_NEEDS_INIT; +#endif +#define PYBIND11_TLS_KEY_CREATE(var) (PyThread_tss_create(&(var)) == 0) +#define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get(&(key)) +#define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set(&(key), (value)) +#define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set(&(key), nullptr) +#define PYBIND11_TLS_FREE(key) PyThread_tss_delete(&(key)) + +/// A smart-pointer-like wrapper around a thread-specific value. get/set of the pointer applies to +/// the current thread only. +template +class thread_specific_storage { +public: + thread_specific_storage() { + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) + if (!PYBIND11_TLS_KEY_CREATE(key_)) { + pybind11_fail( + "thread_specific_storage constructor: could not initialize the TSS key!"); + } + } + + ~thread_specific_storage() { + // This destructor is often called *after* Py_Finalize(). That *SHOULD BE* fine on most + // platforms. The following details what happens when PyThread_tss_free is called in + // CPython. PYBIND11_TLS_FREE is PyThread_tss_free on python 3.7+. On older python, it does + // nothing. PyThread_tss_free calls PyThread_tss_delete and PyMem_RawFree. + // PyThread_tss_delete just calls TlsFree (on Windows) or pthread_key_delete (on *NIX). + // Neither of those have anything to do with CPython internals. PyMem_RawFree *requires* + // that the `key` be allocated with the CPython allocator (as it is by + // PyThread_tss_create). + // However, in GraalPy (as of v25.0 or older), TSS is implemented by Java and this call + // requires a living Python interpreter. +#ifdef GRAALVM_PYTHON + if (Py_IsInitialized() == 0 || _Py_IsFinalizing() != 0) { + return; + } +#endif + PYBIND11_TLS_FREE(key_); + } + + thread_specific_storage(thread_specific_storage const &) = delete; + thread_specific_storage(thread_specific_storage &&) = delete; + thread_specific_storage &operator=(thread_specific_storage const &) = delete; + thread_specific_storage &operator=(thread_specific_storage &&) = delete; + + T *get() const { return reinterpret_cast(PYBIND11_TLS_GET_VALUE(key_)); } + + T &operator*() const { return *get(); } + explicit operator T *() const { return get(); } + explicit operator bool() const { return get() != nullptr; } + + void set(T *val) { PYBIND11_TLS_REPLACE_VALUE(key_, reinterpret_cast(val)); } + void reset(T *p = nullptr) { set(p); } + thread_specific_storage &operator=(T *pval) { + set(pval); + return *this; + } + +private: + PYBIND11_TLS_KEY_INIT(mutable key_) +}; + PYBIND11_NAMESPACE_BEGIN(detail) -constexpr const char *internals_function_record_capsule_name = "pybind11_function_record_capsule"; +// This does NOT actually exist as a module. +#define PYBIND11_DUMMY_MODULE_NAME "pybind11_builtins" // Forward declarations inline PyTypeObject *make_static_property_type(); inline PyTypeObject *make_default_metaclass(); inline PyObject *make_object_base_type(PyTypeObject *metaclass); +inline void translate_exception(std::exception_ptr p); -// The old Python Thread Local Storage (TLS) API is deprecated in Python 3.7 in favor of the new -// Thread Specific Storage (TSS) API. -#if PY_VERSION_HEX >= 0x03070000 -// Avoid unnecessary allocation of `Py_tss_t`, since we cannot use -// `Py_LIMITED_API` anyway. -# if PYBIND11_INTERNALS_VERSION > 4 -# define PYBIND11_TLS_KEY_REF Py_tss_t & -# if defined(__GNUC__) && !defined(__INTEL_COMPILER) -// Clang on macOS warns due to `Py_tss_NEEDS_INIT` not specifying an initializer -// for every field. -# define PYBIND11_TLS_KEY_INIT(var) \ - _Pragma("GCC diagnostic push") /**/ \ - _Pragma("GCC diagnostic ignored \"-Wmissing-field-initializers\"") /**/ \ - Py_tss_t var \ - = Py_tss_NEEDS_INIT; \ - _Pragma("GCC diagnostic pop") -# else -# define PYBIND11_TLS_KEY_INIT(var) Py_tss_t var = Py_tss_NEEDS_INIT; -# endif -# define PYBIND11_TLS_KEY_CREATE(var) (PyThread_tss_create(&(var)) == 0) -# define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get(&(key)) -# define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set(&(key), (value)) -# define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set(&(key), nullptr) -# define PYBIND11_TLS_FREE(key) PyThread_tss_delete(&(key)) -# else -# define PYBIND11_TLS_KEY_REF Py_tss_t * -# define PYBIND11_TLS_KEY_INIT(var) Py_tss_t *var = nullptr; -# define PYBIND11_TLS_KEY_CREATE(var) \ - (((var) = PyThread_tss_alloc()) != nullptr && (PyThread_tss_create((var)) == 0)) -# define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get((key)) -# define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set((key), (value)) -# define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set((key), nullptr) -# define PYBIND11_TLS_FREE(key) PyThread_tss_free(key) -# endif +inline PyThreadState *get_thread_state_unchecked() { +#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) + return PyThreadState_GET(); +#elif PY_VERSION_HEX < 0x030D0000 + return _PyThreadState_UncheckedGet(); #else -// Usually an int but a long on Cygwin64 with Python 3.x -# define PYBIND11_TLS_KEY_REF decltype(PyThread_create_key()) -# define PYBIND11_TLS_KEY_INIT(var) PYBIND11_TLS_KEY_REF var = 0; -# define PYBIND11_TLS_KEY_CREATE(var) (((var) = PyThread_create_key()) != -1) -# define PYBIND11_TLS_GET_VALUE(key) PyThread_get_key_value((key)) -# if defined(PYPY_VERSION) -// On CPython < 3.4 and on PyPy, `PyThread_set_key_value` strangely does not set -// the value if it has already been set. Instead, it must first be deleted and -// then set again. -inline void tls_replace_value(PYBIND11_TLS_KEY_REF key, void *value) { - PyThread_delete_key_value(key); - PyThread_set_key_value(key, value); + return PyThreadState_GetUnchecked(); +#endif } -# define PYBIND11_TLS_DELETE_VALUE(key) PyThread_delete_key_value(key) -# define PYBIND11_TLS_REPLACE_VALUE(key, value) \ - ::pybind11::detail::tls_replace_value((key), (value)) -# else -# define PYBIND11_TLS_DELETE_VALUE(key) PyThread_set_key_value((key), nullptr) -# define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_set_key_value((key), (value)) -# endif -# define PYBIND11_TLS_FREE(key) (void) key + +inline PyInterpreterState *get_interpreter_state_unchecked() { + auto *tstate = get_thread_state_unchecked(); + return tstate ? tstate->interp : nullptr; +} + +inline object get_python_state_dict() { + object state_dict; +#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) + state_dict = reinterpret_borrow(PyEval_GetBuiltins()); +#else + auto *istate = get_interpreter_state_unchecked(); + if (istate) { + state_dict = reinterpret_borrow(PyInterpreterState_GetDict(istate)); + } #endif + if (!state_dict) { + raise_from(PyExc_SystemError, "pybind11::detail::get_python_state_dict() FAILED"); + throw error_already_set(); + } + return state_dict; +} // Python loads modules by default with dlopen with the RTLD_LOCAL flag; under libc++ and possibly // other STLs, this means `typeid(A)` from one module won't equal `typeid(A)` from another module @@ -114,7 +179,7 @@ inline void tls_replace_value(PYBIND11_TLS_KEY_REF key, void *value) { // libstdc++, this doesn't happen: equality and the type_index hash are based on the type name, // which works. If not under a known-good stl, provide our own name-based hash and equality // functions that use the type name. -#if defined(__GLIBCXX__) +#if !defined(_LIBCPP_VERSION) inline bool same_type(const std::type_info &lhs, const std::type_info &rhs) { return lhs == rhs; } using type_hash = std::hash; using type_equal_to = std::equal_to; @@ -124,7 +189,7 @@ inline bool same_type(const std::type_info &lhs, const std::type_info &rhs) { } struct type_hash { - size_t operator()(const std::type_index &t) const { + size_t operator()(const std::type_index &t) const noexcept { size_t hash = 5381; const char *ptr = t.name(); while (auto c = static_cast(*ptr++)) { @@ -135,32 +200,129 @@ struct type_hash { }; struct type_equal_to { - bool operator()(const std::type_index &lhs, const std::type_index &rhs) const { + bool operator()(const std::type_index &lhs, const std::type_index &rhs) const noexcept { return lhs.name() == rhs.name() || std::strcmp(lhs.name(), rhs.name()) == 0; } }; #endif +// For now, we don't bother adding a fancy hash for pointers and just +// let the standard library use the identity hash function if that's +// what it wants to do (e.g., as in libstdc++). +template +using fast_type_map = std::unordered_map; + template using type_map = std::unordered_map; struct override_hash { - inline size_t operator()(const std::pair &v) const { + size_t operator()(const std::pair &v) const noexcept { size_t value = std::hash()(v.first); value ^= std::hash()(v.second) + 0x9e3779b9 + (value << 6) + (value >> 2); return value; } }; +using instance_map = std::unordered_multimap; + +#ifdef Py_GIL_DISABLED +// Wrapper around PyMutex to provide BasicLockable semantics +class pymutex { + friend class pycritical_section; + PyMutex mutex; + +public: + pymutex() : mutex({}) {} + void lock() { PyMutex_Lock(&mutex); } + void unlock() { PyMutex_Unlock(&mutex); } +}; + +class pycritical_section { + pymutex &mutex; +# if PY_VERSION_HEX >= 0x030E00C1 // 3.14.0rc1 + PyCriticalSection cs; +# endif + +public: + explicit pycritical_section(pymutex &m) : mutex(m) { + // PyCriticalSection_BeginMutex was added in Python 3.15.0a1 and backported to 3.14.0rc1 +# if PY_VERSION_HEX >= 0x030E00C1 // 3.14.0rc1 + PyCriticalSection_BeginMutex(&cs, &mutex.mutex); +# else + // Fall back to direct mutex locking for older free-threaded Python versions + mutex.lock(); +# endif + } + ~pycritical_section() { +# if PY_VERSION_HEX >= 0x030E00C1 // 3.14.0rc1 + PyCriticalSection_End(&cs); +# else + mutex.unlock(); +# endif + } + + // Non-copyable and non-movable to prevent double-unlock + pycritical_section(const pycritical_section &) = delete; + pycritical_section &operator=(const pycritical_section &) = delete; + pycritical_section(pycritical_section &&) = delete; + pycritical_section &operator=(pycritical_section &&) = delete; +}; + +// Instance map shards are used to reduce mutex contention in free-threaded Python. +struct instance_map_shard { + instance_map registered_instances; + pymutex mutex; + // alignas(64) would be better, but causes compile errors in macOS before 10.14 (see #5200) + char padding[64 - (sizeof(instance_map) + sizeof(pymutex)) % 64]; +}; + +static_assert(sizeof(instance_map_shard) % 64 == 0, + "instance_map_shard size is not a multiple of 64 bytes"); + +inline uint64_t round_up_to_next_pow2(uint64_t x) { + // Round-up to the next power of two. + // See https://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 + x--; + x |= (x >> 1); + x |= (x >> 2); + x |= (x >> 4); + x |= (x >> 8); + x |= (x >> 16); + x |= (x >> 32); + x++; + return x; +} +#endif + +class loader_life_support; + /// Internal data structure used to track registered instances and types. /// Whenever binary incompatible changes are made to this structure, /// `PYBIND11_INTERNALS_VERSION` must be incremented. struct internals { +#ifdef Py_GIL_DISABLED + pymutex mutex; + pymutex exception_translator_mutex; +#endif +#if PYBIND11_INTERNALS_VERSION >= 12 + // non-normative but fast "hint" for registered_types_cpp. Meant + // to be used as the first level of a two-level lookup: successful + // lookups are correct, but unsuccessful lookups need to try + // registered_types_cpp and then backfill this map if they find + // anything. + fast_type_map registered_types_cpp_fast; +#endif + // std::type_index -> pybind11's type information type_map registered_types_cpp; // PyTypeObject* -> base type_info(s) std::unordered_map> registered_types_py; - std::unordered_multimap registered_instances; // void * -> instance* +#ifdef Py_GIL_DISABLED + std::unique_ptr instance_shards; // void * -> instance* + size_t instance_shards_mask = 0; +#else + instance_map registered_instances; // void * -> instance* +#endif std::unordered_set, override_hash> inactive_override_cache; type_map> direct_conversions; @@ -168,47 +330,68 @@ struct internals { std::forward_list registered_exception_translators; std::unordered_map shared_data; // Custom data to be shared across // extensions -#if PYBIND11_INTERNALS_VERSION == 4 - std::vector unused_loader_patient_stack_remove_at_v5; -#endif - std::forward_list static_strings; // Stores the std::strings backing - // detail::c_str() - PyTypeObject *static_property_type; - PyTypeObject *default_metaclass; - PyObject *instance_base; -#if defined(WITH_THREAD) + std::forward_list static_strings; // Stores the std::strings backing + // detail::c_str() + PyTypeObject *static_property_type = nullptr; + PyTypeObject *default_metaclass = nullptr; + PyObject *instance_base = nullptr; // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined: - PYBIND11_TLS_KEY_INIT(tstate) -# if PYBIND11_INTERNALS_VERSION > 4 - PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key) -# endif // PYBIND11_INTERNALS_VERSION > 4 + thread_specific_storage tstate; +#if PYBIND11_INTERNALS_VERSION <= 11 + thread_specific_storage loader_life_support_tls; // OBSOLETE (PR #5830) +#endif // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined: PyInterpreterState *istate = nullptr; -# if PYBIND11_INTERNALS_VERSION > 4 - // Note that we have to use a std::string to allocate memory to ensure a unique address - // We want unique addresses since we use pointer equality to compare function records - std::string function_record_capsule_name = internals_function_record_capsule_name; -# endif + type_map native_enum_type_map; - internals() = default; + internals() + : static_property_type(make_static_property_type()), + default_metaclass(make_default_metaclass()), istate(get_interpreter_state_unchecked()) { + tstate.set(nullptr); // See PR #5870 + registered_exception_translators.push_front(&translate_exception); +#ifdef Py_GIL_DISABLED + // Scale proportional to the number of cores. 2x is a heuristic to reduce contention. + // Make sure the number isn't unreasonable by limiting it to 16 bits (65K) + auto num_shards = static_cast( + std::min(round_up_to_next_pow2(2 * std::thread::hardware_concurrency()), + std::numeric_limits::max())); + if (num_shards == 0) { + num_shards = 1; + } + instance_shards.reset(new instance_map_shard[num_shards]); + instance_shards_mask = num_shards - 1; +#endif + } internals(const internals &other) = delete; + internals(internals &&other) = delete; internals &operator=(const internals &other) = delete; - ~internals() { -# if PYBIND11_INTERNALS_VERSION > 4 - PYBIND11_TLS_FREE(loader_life_support_tls_key); -# endif // PYBIND11_INTERNALS_VERSION > 4 - - // This destructor is called *after* Py_Finalize() in finalize_interpreter(). - // That *SHOULD BE* fine. The following details what happens when PyThread_tss_free is - // called. PYBIND11_TLS_FREE is PyThread_tss_free on python 3.7+. On older python, it does - // nothing. PyThread_tss_free calls PyThread_tss_delete and PyMem_RawFree. - // PyThread_tss_delete just calls TlsFree (on Windows) or pthread_key_delete (on *NIX). - // Neither of those have anything to do with CPython internals. PyMem_RawFree *requires* - // that the `tstate` be allocated with the CPython allocator. - PYBIND11_TLS_FREE(tstate); - } -#endif + internals &operator=(internals &&other) = delete; + ~internals() = default; +}; + +// the internals struct (above) is shared between all the modules. local_internals are only +// for a single module. Any changes made to internals may require an update to +// PYBIND11_INTERNALS_VERSION, breaking backwards compatibility. local_internals is, by design, +// restricted to a single module. Whether a module has local internals or not should not +// impact any other modules, because the only things accessing the local internals is the +// module that contains them. +struct local_internals { + // It should be safe to use fast_type_map here because this entire + // data structure is scoped to our single module, and thus a single + // DSO and single instance of type_info for any particular type. + fast_type_map registered_types_cpp; + + std::forward_list registered_exception_translators; + PyTypeObject *function_record_py_type = nullptr; +}; + +enum class holder_enum_t : uint8_t { + undefined, + std_unique_ptr, // Default, lacking interop with std::shared_ptr. + std_shared_ptr, // Lacking interop with std::unique_ptr. + smart_holder, // Full std::unique_ptr / std::shared_ptr interop. + custom_holder, }; /// Additional type information which does not fit into the PyTypeObject. @@ -220,12 +403,33 @@ struct type_info { void *(*operator_new)(size_t); void (*init_instance)(instance *, const void *); void (*dealloc)(value_and_holder &v_h); + + // Cross-DSO-safe function pointers, to sidestep cross-DSO RTTI issues + // on platforms like macOS (see PR #5728 for details): + memory::get_guarded_delete_fn get_memory_guarded_delete = memory::get_guarded_delete; + get_trampoline_self_life_support_fn get_trampoline_self_life_support = nullptr; + std::vector implicit_conversions; std::vector> implicit_casts; std::vector *direct_conversions; buffer_info *(*get_buffer)(PyObject *, void *) = nullptr; void *get_buffer_data = nullptr; void *(*module_local_load)(PyObject *, const type_info *) = nullptr; + holder_enum_t holder_enum_v = holder_enum_t::undefined; + +#if PYBIND11_INTERNALS_VERSION >= 12 + // When a type appears in multiple DSOs, + // internals::registered_types_cpp_fast will have multiple distinct + // keys (the std::type_info from each DSO) mapped to the same + // detail::type_info*. We need to keep track of these aliases so that we clean + // them up when our type is deallocated. A linked list is appropriate + // because it is expected to be 1) usually empty and 2) + // when it's not empty, usually very small. See also `struct + // nb_alias_chain` added in + // https://github.com/wjakob/nanobind/commit/b515b1f7f2f4ecc0357818e6201c94a9f4cbfdc2 + std::forward_list alias_chain; +#endif + /* A simple type never occurs as a (direct or indirect) parent * of a class that makes use of multiple inheritance. * A type can be simple even if it has non-simple ancestors as long as it has no descendants. @@ -233,89 +437,43 @@ struct type_info { bool simple_type : 1; /* True if there is no multiple inheritance in this type's inheritance tree */ bool simple_ancestors : 1; - /* for base vs derived holder_type checks */ - bool default_holder : 1; /* true if this is a type registered with py::module_local */ bool module_local : 1; }; -/// On MSVC, debug and release builds are not ABI-compatible! -#if defined(_MSC_VER) && defined(_DEBUG) -# define PYBIND11_BUILD_TYPE "_debug" -#else -# define PYBIND11_BUILD_TYPE "" -#endif - -/// Let's assume that different compilers are ABI-incompatible. -/// A user can manually set this string if they know their -/// compiler is compatible. -#ifndef PYBIND11_COMPILER_TYPE -# if defined(_MSC_VER) -# define PYBIND11_COMPILER_TYPE "_msvc" -# elif defined(__INTEL_COMPILER) -# define PYBIND11_COMPILER_TYPE "_icc" -# elif defined(__clang__) -# define PYBIND11_COMPILER_TYPE "_clang" -# elif defined(__PGI) -# define PYBIND11_COMPILER_TYPE "_pgi" -# elif defined(__MINGW32__) -# define PYBIND11_COMPILER_TYPE "_mingw" -# elif defined(__CYGWIN__) -# define PYBIND11_COMPILER_TYPE "_gcc_cygwin" -# elif defined(__GNUC__) -# define PYBIND11_COMPILER_TYPE "_gcc" -# else -# define PYBIND11_COMPILER_TYPE "_unknown" -# endif -#endif - -/// Also standard libs -#ifndef PYBIND11_STDLIB -# if defined(_LIBCPP_VERSION) -# define PYBIND11_STDLIB "_libcpp" -# elif defined(__GLIBCXX__) || defined(__GLIBCPP__) -# define PYBIND11_STDLIB "_libstdcpp" -# else -# define PYBIND11_STDLIB "" -# endif -#endif - -/// On Linux/OSX, changes in __GXX_ABI_VERSION__ indicate ABI incompatibility. -#ifndef PYBIND11_BUILD_ABI -# if defined(__GXX_ABI_VERSION) -# define PYBIND11_BUILD_ABI "_cxxabi" PYBIND11_TOSTRING(__GXX_ABI_VERSION) -# else -# define PYBIND11_BUILD_ABI "" -# endif -#endif +/// Information stored in a capsule on py::native_enum() types. Since we don't +/// create a type_info record for native enums, we must store here any +/// information we will need about the enum at runtime. +/// +/// If you make backward-incompatible changes to this structure, you must +/// change the `attribute_name()` so that native enums from older version of +/// pybind11 don't have their records reinterpreted. Better would be to keep +/// the changes backward-compatible (i.e., only add new fields at the end) +/// and detect/indicate their presence using the currently-unused `version`. +struct native_enum_record { + const std::type_info *cpptype; + uint32_t size_bytes; + bool is_signed; + const uint8_t version = 1; -#ifndef PYBIND11_INTERNALS_KIND -# if defined(WITH_THREAD) -# define PYBIND11_INTERNALS_KIND "" -# else -# define PYBIND11_INTERNALS_KIND "_without_thread" -# endif -#endif + static const char *attribute_name() { return "__pybind11_native_enum__"; } +}; #define PYBIND11_INTERNALS_ID \ "__pybind11_internals_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ - PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB PYBIND11_BUILD_ABI \ - PYBIND11_BUILD_TYPE "__" + PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE PYBIND11_PLATFORM_ABI_ID "__" #define PYBIND11_MODULE_LOCAL_ID \ "__pybind11_module_local_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ - PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB PYBIND11_BUILD_ABI \ - PYBIND11_BUILD_TYPE "__" - -/// Each module locally stores a pointer to the `internals` data. The data -/// itself is shared among modules with the same `PYBIND11_INTERNALS_ID`. -inline internals **&get_internals_pp() { - static internals **internals_pp = nullptr; - return internals_pp; -} + PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE PYBIND11_PLATFORM_ABI_ID "__" -// forward decl -inline void translate_exception(std::exception_ptr); +/// We use this to figure out if there are or have been multiple subinterpreters active at any +/// point. This must never go from true to false while any interpreter may be running in any +/// thread! +inline std::atomic_bool &has_seen_non_main_interpreter() { + static std::atomic_bool multi(false); + return multi; +} template >::value, int> = 0> @@ -342,7 +500,7 @@ inline bool raise_err(PyObject *exc_type, const char *msg) { raise_from(exc_type, msg); return true; } - PyErr_SetString(exc_type, msg); + set_error(exc_type, msg); return false; } @@ -421,132 +579,453 @@ inline void translate_local_exception(std::exception_ptr p) { } #endif -/// Return a reference to the current `internals` data -PYBIND11_NOINLINE internals &get_internals() { - auto **&internals_pp = get_internals_pp(); - if (internals_pp && *internals_pp) { - return **internals_pp; - } +// Sentinel value for the `dtor` parameter of `atomic_get_or_create_in_state_dict`. +// Indicates no destructor was explicitly provided (distinct from nullptr, which means "leak"). +#define PYBIND11_DTOR_USE_DELETE (reinterpret_cast(1)) -#if defined(WITH_THREAD) -# if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) - gil_scoped_acquire gil; -# else - // Ensure that the GIL is held since we will need to make Python calls. - // Cannot use py::gil_scoped_acquire here since that constructor calls get_internals. - struct gil_scoped_acquire_local { - gil_scoped_acquire_local() : state(PyGILState_Ensure()) {} - gil_scoped_acquire_local(const gil_scoped_acquire_local &) = delete; - gil_scoped_acquire_local &operator=(const gil_scoped_acquire_local &) = delete; - ~gil_scoped_acquire_local() { PyGILState_Release(state); } - const PyGILState_STATE state; - } gil; -# endif -#endif - error_scope err_scope; +// Get or create per-storage capsule in the current interpreter's state dict. +// - The storage is interpreter-dependent: different interpreters will have different storage. +// This is important when using multiple-interpreters, to avoid sharing unshareable objects +// between interpreters. +// - There is one storage per `key` in an interpreter and it is accessible between all extensions +// in the same interpreter. +// - The life span of the storage is tied to the interpreter: it will be kept alive until the +// interpreter shuts down. +// +// Use test-and-set pattern with `PyDict_SetDefault` for thread-safe concurrent access. +// WARNING: There can be multiple threads creating the storage at the same time, while only one +// will succeed in inserting its capsule into the dict. Therefore, the deleter will be +// used to clean up the storage of the unused capsules. +// +// Returns: pair of (pointer to storage, bool indicating if newly created). +// The bool follows std::map::insert convention: true = created, false = existed. +// `dtor`: optional destructor called when the interpreter shuts down. +// - If not provided: the storage will be deleted using `delete`. +// - If nullptr: the storage will be leaked (useful for singletons that outlive the interpreter). +// - If a function: that function will be called with the capsule object. +template +std::pair atomic_get_or_create_in_state_dict(const char *key, + void (*dtor)(PyObject *) + = PYBIND11_DTOR_USE_DELETE) { + error_scope err_scope; // preserve any existing Python error states - PYBIND11_STR_TYPE id(PYBIND11_INTERNALS_ID); - auto builtins = handle(PyEval_GetBuiltins()); - if (builtins.contains(id) && isinstance(builtins[id])) { - internals_pp = static_cast(capsule(builtins[id])); + auto state_dict = reinterpret_borrow(get_python_state_dict()); + PyObject *capsule_obj = nullptr; + bool created = false; - // We loaded builtins through python's builtins, which means that our `error_already_set` - // and `builtin_exception` may be different local classes than the ones set up in the - // initial exception translator, below, so add another for our local exception classes. + // Try to get existing storage (fast path). + capsule_obj = dict_getitemstring(state_dict.ptr(), key); + if (capsule_obj == nullptr) { + if (PyErr_Occurred()) { + throw error_already_set(); + } + // Storage doesn't exist yet, create a new one. + // Use unique_ptr for exception safety: if capsule creation throws, the storage is + // automatically deleted. + auto storage_ptr = std::unique_ptr(new Payload{}); + auto new_capsule + = capsule(storage_ptr.get(), + // The destructor will be called when the capsule is GC'ed. + // If the insert below fails (entry already in the dict), then this + // destructor will be called on the newly created capsule at the end of this + // function, and we want to just release this memory. + /*destructor=*/[](void *v) { delete static_cast(v); }); + // At this point, the capsule object is created successfully. + // Release the unique_ptr and let the capsule object own the storage to avoid double-free. + (void) storage_ptr.release(); + + // Use `PyDict_SetDefault` for atomic test-and-set: + // - If key doesn't exist, inserts our capsule and returns it. + // - If key exists (another thread inserted first), returns the existing value. + // This is thread-safe because `PyDict_SetDefault` will hold a lock on the dict. // - // libstdc++ doesn't require this (types there are identified only by name) - // libc++ with CPython doesn't require this (types are explicitly exported) - // libc++ with PyPy still need it, awaiting further investigation -#if !defined(__GLIBCXX__) - (*internals_pp)->registered_exception_translators.push_front(&translate_local_exception); + // NOTE: Here we use `PyDict_SetDefault` instead of `PyDict_SetDefaultRef` because the + // capsule is kept alive until interpreter shutdown, so we do not need to handle + // incref and decref here. + capsule_obj = dict_setdefaultstring(state_dict.ptr(), key, new_capsule.ptr()); + if (capsule_obj == nullptr) { + throw error_already_set(); + } + created = (capsule_obj == new_capsule.ptr()); + // - If key already existed, our `new_capsule` is not inserted, it will be destructed when + // going out of scope here, and will call the destructor set above. + // - Otherwise, our `new_capsule` is now in the dict, and it owns the storage and the state + // dict will incref it. We need to set the caller's destructor on it, which will be + // called when the interpreter shuts down. + if (created && dtor != PYBIND11_DTOR_USE_DELETE) { + if (PyCapsule_SetDestructor(capsule_obj, dtor) < 0) { + throw error_already_set(); + } + } + } + + // Get the storage pointer from the capsule. + void *raw_ptr = PyCapsule_GetPointer(capsule_obj, /*name=*/nullptr); + if (!raw_ptr) { + raise_from(PyExc_SystemError, + "pybind11::detail::atomic_get_or_create_in_state_dict() FAILED"); + throw error_already_set(); + } + return std::pair(static_cast(raw_ptr), created); +} + +#undef PYBIND11_DTOR_USE_DELETE + +template +class internals_pp_manager { +public: + using on_fetch_function = void(InternalsType *); + + static internals_pp_manager &get_instance(char const *id, on_fetch_function *on_fetch) { + static internals_pp_manager instance(id, on_fetch); + return instance; + } + + /// Get the current pointer-to-pointer, allocating it if it does not already exist. May + /// acquire the GIL. Will never return nullptr. + std::unique_ptr *get_pp() { +#ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT + if (has_seen_non_main_interpreter()) { + // Whenever the interpreter changes on the current thread we need to invalidate the + // internals_pp so that it can be pulled from the interpreter's state dict. That is + // slow, so we use the current PyThreadState to check if it is necessary. + auto *tstate = get_thread_state_unchecked(); + if (!tstate || tstate->interp != last_istate_tls()) { + gil_scoped_acquire_simple gil; + if (!tstate) { + tstate = get_thread_state_unchecked(); + } + last_istate_tls() = tstate->interp; + internals_p_tls() = get_or_create_pp_in_state_dict(); + } + return internals_p_tls(); + } #endif - } else { - if (!internals_pp) { - internals_pp = new internals *(); + if (!internals_singleton_pp_) { + gil_scoped_acquire_simple gil; + internals_singleton_pp_ = get_or_create_pp_in_state_dict(); } - auto *&internals_ptr = *internals_pp; - internals_ptr = new internals(); -#if defined(WITH_THREAD) + return internals_singleton_pp_; + } - PyThreadState *tstate = PyThreadState_Get(); - // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) - if (!PYBIND11_TLS_KEY_CREATE(internals_ptr->tstate)) { - pybind11_fail("get_internals: could not successfully initialize the tstate TSS key!"); + /// Drop all the references we're currently holding. + void unref() { +#ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT + if (has_seen_non_main_interpreter()) { + last_istate_tls() = nullptr; + internals_p_tls() = nullptr; + return; } - PYBIND11_TLS_REPLACE_VALUE(internals_ptr->tstate, tstate); +#endif + internals_singleton_pp_ = nullptr; + } -# if PYBIND11_INTERNALS_VERSION > 4 - // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) - if (!PYBIND11_TLS_KEY_CREATE(internals_ptr->loader_life_support_tls_key)) { - pybind11_fail("get_internals: could not successfully initialize the " - "loader_life_support TSS key!"); + void destroy() { +#ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT + if (has_seen_non_main_interpreter()) { + auto *tstate = get_thread_state_unchecked(); + // this could be called without an active interpreter, just use what was cached + if (!tstate || tstate->interp == last_istate_tls()) { + auto tpp = internals_p_tls(); + { + std::lock_guard lock(pp_set_mutex_); + pps_have_created_content_.erase(tpp); // untrack deleted pp + } + delete tpp; // may call back into Python + } + unref(); + return; } -# endif - internals_ptr->istate = tstate->interp; #endif - builtins[id] = capsule(internals_pp); - internals_ptr->registered_exception_translators.push_front(&translate_exception); - internals_ptr->static_property_type = make_static_property_type(); - internals_ptr->default_metaclass = make_default_metaclass(); - internals_ptr->instance_base = make_object_base_type(internals_ptr->default_metaclass); + { + std::lock_guard lock(pp_set_mutex_); + pps_have_created_content_.erase(internals_singleton_pp_); // untrack deleted pp + } + delete internals_singleton_pp_; // may call back into Python + unref(); } - return **internals_pp; -} -// the internals struct (above) is shared between all the modules. local_internals are only -// for a single module. Any changes made to internals may require an update to -// PYBIND11_INTERNALS_VERSION, breaking backwards compatibility. local_internals is, by design, -// restricted to a single module. Whether a module has local internals or not should not -// impact any other modules, because the only things accessing the local internals is the -// module that contains them. -struct local_internals { - type_map registered_types_cpp; - std::forward_list registered_exception_translators; -#if defined(WITH_THREAD) && PYBIND11_INTERNALS_VERSION == 4 - - // For ABI compatibility, we can't store the loader_life_support TLS key in - // the `internals` struct directly. Instead, we store it in `shared_data` and - // cache a copy in `local_internals`. If we allocated a separate TLS key for - // each instance of `local_internals`, we could end up allocating hundreds of - // TLS keys if hundreds of different pybind11 modules are loaded (which is a - // plausible number). - PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key) - - // Holds the shared TLS key for the loader_life_support stack. - struct shared_loader_life_support_data { - PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key) - shared_loader_life_support_data() { - // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) - if (!PYBIND11_TLS_KEY_CREATE(loader_life_support_tls_key)) { - pybind11_fail("local_internals: could not successfully initialize the " - "loader_life_support TLS key!"); + void create_pp_content_once(std::unique_ptr *const pp) { + // Assume the GIL is held here. May call back into Python. We cannot hold the lock with our + // mutex here. So there may be multiple threads creating the content at the same time. Only + // one will install its content to pp below. Others will be freed when going out of scope. + auto tmp = std::unique_ptr(new InternalsType()); + + { + // Lock scope must not include Python calls, which may require the GIL and cause + // deadlocks. + std::lock_guard lock(pp_set_mutex_); + + if (*pp) { + // Already created in another thread. + return; } + + // At this point, pp->get() is nullptr. + // The content is either not yet created, or was previously destroyed via pp->reset(). + + // Detect re-creation of internals after destruction during interpreter shutdown. + // If pybind11 code (e.g., tp_traverse/tp_clear calling py::cast) runs after internals + // have been destroyed, a new empty internals would be created, causing type lookup + // failures. See also get_or_create_pp_in_state_dict() comments. + if (pps_have_created_content_.find(pp) != pps_have_created_content_.end()) { + pybind11_fail( + "pybind11::detail::internals_pp_manager::create_pp_content_once() " + "FAILED: reentrant call detected while fetching pybind11 internals!"); + } + + // Each interpreter can only create its internals once. + pps_have_created_content_.insert(pp); + // Install the created content. + pp->swap(tmp); } - // We can't help but leak the TLS key, because Python never unloads extension modules. - }; + } - local_internals() { - auto &internals = get_internals(); - // Get or create the `loader_life_support_stack_key`. - auto &ptr = internals.shared_data["_life_support"]; - if (!ptr) { - ptr = new shared_loader_life_support_data; +private: + internals_pp_manager(char const *id, on_fetch_function *on_fetch) + : holder_id_(id), on_fetch_(on_fetch) {} + + std::unique_ptr *get_or_create_pp_in_state_dict() { + // The `unique_ptr` is intentionally leaked on interpreter shutdown. + // Once an instance is created, it will never be deleted until the process exits (compare + // to interpreter shutdown in multiple-interpreter scenarios). + // We cannot guarantee the destruction order of capsules in the interpreter state dict on + // interpreter shutdown, so deleting internals too early could cause undefined behavior + // when other pybind11 objects access `get_internals()` during finalization (which would + // recreate empty internals). See also create_pp_content_once() above. + // See https://github.com/pybind/pybind11/pull/5958#discussion_r2717645230. + auto result = atomic_get_or_create_in_state_dict>( + holder_id_, /*dtor=*/nullptr /* leak the capsule content */); + auto *pp = result.first; + bool created = result.second; + // Only call on_fetch_ when fetching existing internals, not when creating new ones. + if (!created && on_fetch_ && pp) { + on_fetch_(pp->get()); } - loader_life_support_tls_key - = static_cast(ptr)->loader_life_support_tls_key; + return pp; + } + +#ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT + static PyInterpreterState *&last_istate_tls() { + static thread_local PyInterpreterState *last_istate = nullptr; + return last_istate; + } + + static std::unique_ptr *&internals_p_tls() { + static thread_local std::unique_ptr *internals_p = nullptr; + return internals_p; } -#endif // defined(WITH_THREAD) && PYBIND11_INTERNALS_VERSION == 4 +#endif + + char const *holder_id_ = nullptr; + on_fetch_function *on_fetch_ = nullptr; + // Pointer-to-pointer to the singleton internals for the first seen interpreter (may not be the + // main interpreter) + std::unique_ptr *internals_singleton_pp_ = nullptr; + + // Track pointer-to-pointers whose internals have been created, to detect re-entrancy. + // Use instance member over static due to singleton pattern of this class. + std::unordered_set *> pps_have_created_content_; + std::mutex pp_set_mutex_; }; +// If We loaded the internals through `state_dict`, our `error_already_set` +// and `builtin_exception` may be different local classes than the ones set up in the +// initial exception translator, below, so add another for our local exception classes. +// +// libstdc++ doesn't require this (types there are identified only by name) +// libc++ with CPython doesn't require this (types are explicitly exported) +// libc++ with PyPy still need it, awaiting further investigation +#if !defined(__GLIBCXX__) +inline void check_internals_local_exception_translator(internals *internals_ptr) { + if (internals_ptr) { + for (auto et : internals_ptr->registered_exception_translators) { + if (et == &translate_local_exception) { + return; + } + } + internals_ptr->registered_exception_translators.push_front(&translate_local_exception); + } +} +#endif + +inline internals_pp_manager &get_internals_pp_manager() { +#if defined(__GLIBCXX__) +# define ON_FETCH_FN nullptr +#else +# define ON_FETCH_FN &check_internals_local_exception_translator +#endif + return internals_pp_manager::get_instance(PYBIND11_INTERNALS_ID, ON_FETCH_FN); +#undef ON_FETCH_FN +} + +/// Return a reference to the current `internals` data +PYBIND11_NOINLINE internals &get_internals() { + auto &ppmgr = get_internals_pp_manager(); + auto *pp = ppmgr.get_pp(); + if (!pp) { + pybind11_fail("get_internals: get_pp() returned nullptr"); + } + auto &internals_ptr = *pp; + if (!internals_ptr) { + // Slow path, something needs fetched from the state dict or created + gil_scoped_acquire_simple gil; + error_scope err_scope; + + ppmgr.create_pp_content_once(&internals_ptr); + + if (!internals_ptr) { + pybind11_fail("get_internals: create_pp_content_once() produced nullptr"); + } + if (!internals_ptr->instance_base) { + // This calls get_internals, so cannot be called from within the internals constructor + // called above because internals_ptr must be set before get_internals is called again + internals_ptr->instance_base = make_object_base_type(internals_ptr->default_metaclass); + } + } + return *internals_ptr; +} + +/// Return the PyObject* for the internals capsule (borrowed reference). +/// Returns nullptr if the capsule doesn't exist yet. +inline PyObject *get_internals_capsule() { + auto state_dict = reinterpret_borrow(get_python_state_dict()); + return dict_getitemstring(state_dict.ptr(), PYBIND11_INTERNALS_ID); +} + +/// Return the key used for local_internals in the state dict. +/// This function ensures a consistent key is used across all call sites within the same +/// compilation unit. The key includes the address of a static variable to make it unique per +/// module (DSO), matching the behavior of get_local_internals_pp_manager(). +inline const std::string &get_local_internals_key() { + static const std::string key + = PYBIND11_MODULE_LOCAL_ID + std::to_string(reinterpret_cast(&key)); + return key; +} + +/// Return the PyObject* for the local_internals capsule (borrowed reference). +/// Returns nullptr if the capsule doesn't exist yet. +inline PyObject *get_local_internals_capsule() { + const auto &key = get_local_internals_key(); + auto state_dict = reinterpret_borrow(get_python_state_dict()); + return dict_getitemstring(state_dict.ptr(), key.c_str()); +} + +inline void ensure_internals() { + pybind11::detail::get_internals_pp_manager().unref(); +#ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT + if (PyInterpreterState_Get() != PyInterpreterState_Main()) { + has_seen_non_main_interpreter() = true; + } +#endif + pybind11::detail::get_internals(); +} + +inline internals_pp_manager &get_local_internals_pp_manager() { + // Use the address of a static variable as part of the key, so that the value is uniquely tied + // to where the module is loaded in memory + return internals_pp_manager::get_instance(get_local_internals_key().c_str(), + nullptr); +} + /// Works like `get_internals`, but for things which are locally registered. inline local_internals &get_local_internals() { - // Current static can be created in the interpreter finalization routine. If the later will be - // destroyed in another static variable destructor, creation of this static there will cause - // static deinitialization fiasco. In order to avoid it we avoid destruction of the - // local_internals static. One can read more about the problem and current solution here: - // https://google.github.io/styleguide/cppguide.html#Static_and_Global_Variables - static auto *locals = new local_internals(); - return *locals; + auto &ppmgr = get_local_internals_pp_manager(); + auto &internals_ptr = *ppmgr.get_pp(); + if (!internals_ptr) { + gil_scoped_acquire_simple gil; + error_scope err_scope; + + ppmgr.create_pp_content_once(&internals_ptr); + } + return *internals_ptr; +} + +#ifdef Py_GIL_DISABLED +# define PYBIND11_LOCK_INTERNALS(internals) pycritical_section lock((internals).mutex) +#else +# define PYBIND11_LOCK_INTERNALS(internals) +#endif + +template +inline auto with_internals(const F &cb) -> decltype(cb(get_internals())) { + auto &internals = get_internals(); + PYBIND11_LOCK_INTERNALS(internals); + return cb(internals); +} + +template +inline void with_internals_if_internals(const F &cb) { + auto &ppmgr = get_internals_pp_manager(); + auto &internals_ptr = *ppmgr.get_pp(); + if (internals_ptr) { + auto &internals = *internals_ptr; + PYBIND11_LOCK_INTERNALS(internals); + cb(internals); + } +} + +template +inline auto with_exception_translators(const F &cb) + -> decltype(cb(get_internals().registered_exception_translators, + get_local_internals().registered_exception_translators)) { + auto &internals = get_internals(); +#ifdef Py_GIL_DISABLED + pycritical_section lock((internals).exception_translator_mutex); +#endif + auto &local_internals = get_local_internals(); + return cb(internals.registered_exception_translators, + local_internals.registered_exception_translators); +} + +inline std::uint64_t mix64(std::uint64_t z) { + // David Stafford's variant 13 of the MurmurHash3 finalizer popularized + // by the SplitMix PRNG. + // https://zimbry.blogspot.com/2011/09/better-bit-mixing-improving-on.html + z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9; + z = (z ^ (z >> 27)) * 0x94d049bb133111eb; + return z ^ (z >> 31); +} + +template +inline auto with_instance_map(const void *ptr, const F &cb) + -> decltype(cb(std::declval())) { + auto &internals = get_internals(); + +#ifdef Py_GIL_DISABLED + // Hash address to compute shard, but ignore low bits. We'd like allocations + // from the same thread/core to map to the same shard and allocations from + // other threads/cores to map to other shards. Using the high bits is a good + // heuristic because memory allocators often have a per-thread + // arena/superblock/segment from which smaller allocations are served. + auto addr = reinterpret_cast(ptr); + auto hash = mix64(static_cast(addr >> 20)); + auto idx = static_cast(hash & internals.instance_shards_mask); + + auto &shard = internals.instance_shards[idx]; + std::unique_lock lock(shard.mutex); + return cb(shard.registered_instances); +#else + (void) ptr; + return cb(internals.registered_instances); +#endif +} + +// Returns the number of registered instances for testing purposes. The result may not be +// consistent if other threads are registering or unregistering instances concurrently. +inline size_t num_registered_instances() { + auto &internals = get_internals(); +#ifdef Py_GIL_DISABLED + size_t count = 0; + for (size_t i = 0; i <= internals.instance_shards_mask; ++i) { + auto &shard = internals.instance_shards[i]; + std::unique_lock lock(shard.mutex); + count += shard.registered_instances.size(); + } + return count; +#else + return internals.registered_instances.size(); +#endif } /// Constructs a std::string with the given arguments, stores it in `internals`, and returns its @@ -555,45 +1034,33 @@ inline local_internals &get_local_internals() { /// suitable for c-style strings needed by Python internals (such as PyTypeObject's tp_name). template const char *c_str(Args &&...args) { - auto &strings = get_internals().static_strings; + // GCC 4.8 doesn't like parameter unpack within lambda capture, so use + // PYBIND11_LOCK_INTERNALS. + auto &internals = get_internals(); + PYBIND11_LOCK_INTERNALS(internals); + auto &strings = internals.static_strings; strings.emplace_front(std::forward(args)...); return strings.front().c_str(); } -inline const char *get_function_record_capsule_name() { -#if PYBIND11_INTERNALS_VERSION > 4 - return get_internals().function_record_capsule_name.c_str(); -#else - return nullptr; -#endif -} - -// Determine whether or not the following capsule contains a pybind11 function record. -// Note that we use `internals` to make sure that only ABI compatible records are touched. -// -// This check is currently used in two places: -// - An important optimization in functional.h to avoid overhead in C++ -> Python -> C++ -// - The sibling feature of cpp_function to allow overloads -inline bool is_function_record_capsule(const capsule &cap) { - // Pointer equality as we rely on internals() to ensure unique pointers - return cap.name() == get_function_record_capsule_name(); -} - PYBIND11_NAMESPACE_END(detail) /// Returns a named pointer that is shared among all extension modules (using the same /// pybind11 version) running in the current interpreter. Names starting with underscores /// are reserved for internal usage. Returns `nullptr` if no matching entry was found. PYBIND11_NOINLINE void *get_shared_data(const std::string &name) { - auto &internals = detail::get_internals(); - auto it = internals.shared_data.find(name); - return it != internals.shared_data.end() ? it->second : nullptr; + return detail::with_internals([&](detail::internals &internals) { + auto it = internals.shared_data.find(name); + return it != internals.shared_data.end() ? it->second : nullptr; + }); } /// Set the shared data that can be later recovered by `get_shared_data()`. PYBIND11_NOINLINE void *set_shared_data(const std::string &name, void *data) { - detail::get_internals().shared_data[name] = data; - return data; + return detail::with_internals([&](detail::internals &internals) { + internals.shared_data[name] = data; + return data; + }); } /// Returns a typed reference to a shared data entry (by using `get_shared_data()`) if @@ -601,14 +1068,15 @@ PYBIND11_NOINLINE void *set_shared_data(const std::string &name, void *data) { /// added to the shared data under the given name and a reference to it is returned. template T &get_or_create_shared_data(const std::string &name) { - auto &internals = detail::get_internals(); - auto it = internals.shared_data.find(name); - T *ptr = (T *) (it != internals.shared_data.end() ? it->second : nullptr); - if (!ptr) { - ptr = new T(); - internals.shared_data[name] = ptr; - } - return *ptr; + return *detail::with_internals([&](detail::internals &internals) { + auto it = internals.shared_data.find(name); + T *ptr = (T *) (it != internals.shared_data.end() ? it->second : nullptr); + if (!ptr) { + ptr = new T(); + internals.shared_data[name] = ptr; + } + return ptr; + }); } PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/detail/native_enum_data.h b/lib/pybind11/include/pybind11/detail/native_enum_data.h new file mode 100644 index 0000000..6770378 --- /dev/null +++ b/lib/pybind11/include/pybind11/detail/native_enum_data.h @@ -0,0 +1,227 @@ +// Copyright (c) 2022-2025 The pybind Community. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#pragma once + +#include "../pytypes.h" +#include "common.h" +#include "internals.h" + +#include +#include +#include +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +// This is a separate function only to enable easy unit testing. +inline std::string +native_enum_missing_finalize_error_message(const std::string &enum_name_encoded) { + return "pybind11::native_enum<...>(\"" + enum_name_encoded + "\", ...): MISSING .finalize()"; +} + +// Internals for pybind11::native_enum; one native_enum_data object exists +// inside each pybind11::native_enum and lives only for the duration of the +// native_enum binding statement. +class native_enum_data { +public: + native_enum_data(handle parent_scope_, + const char *enum_name, + const char *native_type_name, + const char *class_doc, + const native_enum_record &enum_record_) + : enum_name_encoded{enum_name}, native_type_name_encoded{native_type_name}, + enum_type_index{*enum_record_.cpptype}, + parent_scope(reinterpret_borrow(parent_scope_)), enum_name{enum_name}, + native_type_name{native_type_name}, class_doc(class_doc), export_values_flag{false}, + finalize_needed{false} { + // Create the enum record capsule. It will be installed on the enum + // type object during finalize(). Its destructor removes the enum + // mapping from our internals, so that we won't try to convert to an + // enum type that's been destroyed. + enum_record = capsule( + new native_enum_record{enum_record_}, + native_enum_record::attribute_name(), + +[](void *record_) { + auto *record = static_cast(record_); + with_internals([&](internals &internals) { + internals.native_enum_type_map.erase(*record->cpptype); + }); + delete record; + }); + } + + void finalize(); + + native_enum_data(const native_enum_data &) = delete; + native_enum_data &operator=(const native_enum_data &) = delete; + +#if !defined(NDEBUG) + // This dtor cannot easily be unit tested because it terminates the process. + ~native_enum_data() { + if (finalize_needed) { + pybind11_fail(native_enum_missing_finalize_error_message(enum_name_encoded)); + } + } +#endif + +protected: + void disarm_finalize_check(const char *error_context) { + if (!finalize_needed) { + pybind11_fail("pybind11::native_enum<...>(\"" + enum_name_encoded + + "\"): " + error_context); + } + finalize_needed = false; + } + + void arm_finalize_check() { + assert(!finalize_needed); // Catch redundant calls. + finalize_needed = true; + } + + std::string enum_name_encoded; + std::string native_type_name_encoded; + std::type_index enum_type_index; + +private: + object parent_scope; + str enum_name; + str native_type_name; + std::string class_doc; + capsule enum_record; + +protected: + list members; + list member_docs; + bool export_values_flag : 1; // Attention: It is best to keep the bools together. + +private: + bool finalize_needed : 1; +}; + +inline handle +global_internals_native_enum_type_map_get_item(const std::type_index &enum_type_index) { + return with_internals([&](internals &internals) { + auto found = internals.native_enum_type_map.find(enum_type_index); + if (found != internals.native_enum_type_map.end()) { + return handle(found->second); + } + return handle(); + }); +} + +inline bool +global_internals_native_enum_type_map_contains(const std::type_index &enum_type_index) { + return with_internals([&](internals &internals) { + return internals.native_enum_type_map.count(enum_type_index) != 0; + }); +} + +inline object import_or_getattr(const std::string &fully_qualified_name, + const std::string &append_to_exception_message) { + std::istringstream stream(fully_qualified_name); + std::string part; + + if (!std::getline(stream, part, '.') || part.empty()) { + std::string msg = "Invalid fully-qualified name `"; + msg += fully_qualified_name; + msg += "`"; + msg += append_to_exception_message; + throw value_error(msg); + } + + auto curr_scope = reinterpret_steal(PyImport_ImportModule(part.c_str())); + if (!curr_scope) { + std::string msg = "Failed to import top-level module `"; + msg += part; + msg += "`"; + msg += append_to_exception_message; + raise_from(PyExc_ImportError, msg.c_str()); + throw error_already_set(); + } + + // Now recursively getattr or import remaining parts + std::string curr_path = part; + while (std::getline(stream, part, '.')) { + if (part.empty()) { + std::string msg = "Invalid fully-qualified name `"; + msg += fully_qualified_name; + msg += "`"; + msg += append_to_exception_message; + throw value_error(msg); + } + std::string next_path = curr_path; + next_path += "."; + next_path += part; + auto next_scope + = reinterpret_steal(PyObject_GetAttrString(curr_scope.ptr(), part.c_str())); + if (!next_scope) { + error_fetch_and_normalize stored_getattr_error("getattr"); + // Try importing the next level + next_scope = reinterpret_steal(PyImport_ImportModule(next_path.c_str())); + if (!next_scope) { + error_fetch_and_normalize stored_import_error("import"); + std::string msg = "Failed to import or getattr `"; + msg += part; + msg += "` from `"; + msg += curr_path; + msg += "`"; + msg += append_to_exception_message; + msg += "\n-------- getattr exception --------\n"; + msg += stored_getattr_error.error_string(); + msg += "\n-------- import exception --------\n"; + msg += stored_import_error.error_string(); + throw import_error(msg.c_str()); + } + } + curr_scope = next_scope; + curr_path = next_path; + } + return curr_scope; +} + +inline void native_enum_data::finalize() { + disarm_finalize_check("DOUBLE finalize"); + if (hasattr(parent_scope, enum_name)) { + pybind11_fail("pybind11::native_enum<...>(\"" + enum_name_encoded + + "\"): an object with that name is already defined"); + } + auto py_enum_type = import_or_getattr(native_type_name, " (native_type_name)"); + auto py_enum = py_enum_type(enum_name, members); + object module_name = get_module_name_if_available(parent_scope); + if (module_name) { + py_enum.attr("__module__") = module_name; + } + if (hasattr(parent_scope, "__qualname__")) { + const auto parent_qualname = parent_scope.attr("__qualname__").cast(); + py_enum.attr("__qualname__") = str(parent_qualname + "." + enum_name.cast()); + } + parent_scope.attr(enum_name) = py_enum; + if (export_values_flag) { + for (auto member : members) { + auto member_name = member[int_(0)]; + if (hasattr(parent_scope, member_name)) { + pybind11_fail("pybind11::native_enum<...>(\"" + enum_name_encoded + "\").value(\"" + + member_name.cast() + + "\"): an object with that name is already defined"); + } + parent_scope.attr(member_name) = py_enum[member_name]; + } + } + if (!class_doc.empty()) { + py_enum.attr("__doc__") = class_doc.c_str(); + } + for (auto doc : member_docs) { + py_enum[doc[int_(0)]].attr("__doc__") = doc[int_(1)]; + } + + py_enum.attr(native_enum_record::attribute_name()) = enum_record; + with_internals([&](internals &internals) { + internals.native_enum_type_map[enum_type_index] = py_enum.ptr(); + }); +} + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/detail/pybind11_namespace_macros.h b/lib/pybind11/include/pybind11/detail/pybind11_namespace_macros.h new file mode 100644 index 0000000..6f74bf8 --- /dev/null +++ b/lib/pybind11/include/pybind11/detail/pybind11_namespace_macros.h @@ -0,0 +1,82 @@ +// Copyright (c) 2016-2025 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#pragma once + +// PLEASE DO NOT ADD ANY INCLUDES HERE + +// Define some generic pybind11 helper macros for warning management. +// +// Note that compiler-specific push/pop pairs are baked into the +// PYBIND11_NAMESPACE_BEGIN/PYBIND11_NAMESPACE_END pair of macros. Therefore manual +// PYBIND11_WARNING_PUSH/PYBIND11_WARNING_POP are usually only needed in `#include` sections. +// +// If you find you need to suppress a warning, please try to make the suppression as local as +// possible using these macros. Please also be sure to push/pop with the pybind11 macros. Please +// only use compiler specifics if you need to check specific versions, e.g. Apple Clang vs. vanilla +// Clang. +#if defined(__INTEL_COMPILER) +# define PYBIND11_COMPILER_INTEL +# define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(warning push) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(warning pop) +#elif defined(__clang__) +# define PYBIND11_COMPILER_CLANG +# define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(clang diagnostic push) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(clang diagnostic pop) +#elif defined(__GNUC__) +# define PYBIND11_COMPILER_GCC +# define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(GCC diagnostic push) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(GCC diagnostic pop) +#elif defined(_MSC_VER) // Must be after the clang branch because clang-cl also defines _MSC_VER +# define PYBIND11_COMPILER_MSVC +# define PYBIND11_PRAGMA(...) __pragma(__VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(warning(push)) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(warning(pop)) +#endif + +#ifdef PYBIND11_COMPILER_MSVC +# define PYBIND11_WARNING_DISABLE_MSVC(name) PYBIND11_PRAGMA(warning(disable : name)) +#else +# define PYBIND11_WARNING_DISABLE_MSVC(name) +#endif + +#ifdef PYBIND11_COMPILER_CLANG +# define PYBIND11_WARNING_DISABLE_CLANG(name) PYBIND11_PRAGMA(clang diagnostic ignored name) +#else +# define PYBIND11_WARNING_DISABLE_CLANG(name) +#endif + +#ifdef PYBIND11_COMPILER_GCC +# define PYBIND11_WARNING_DISABLE_GCC(name) PYBIND11_PRAGMA(GCC diagnostic ignored name) +#else +# define PYBIND11_WARNING_DISABLE_GCC(name) +#endif + +#ifdef PYBIND11_COMPILER_INTEL +# define PYBIND11_WARNING_DISABLE_INTEL(name) PYBIND11_PRAGMA(warning disable name) +#else +# define PYBIND11_WARNING_DISABLE_INTEL(name) +#endif + +#define PYBIND11_NAMESPACE_BEGIN(name) \ + namespace name { \ + PYBIND11_WARNING_PUSH + +#define PYBIND11_NAMESPACE_END(name) \ + PYBIND11_WARNING_POP \ + } + +// Robust support for some features and loading modules compiled against different pybind versions +// requires forcing hidden visibility on pybind code, so we enforce this by setting the attribute +// on the main `pybind11` namespace. +#if !defined(PYBIND11_NAMESPACE) +# if defined(__GNUG__) && !defined(_WIN32) +# define PYBIND11_NAMESPACE pybind11 __attribute__((visibility("hidden"))) +# else +# define PYBIND11_NAMESPACE pybind11 +# endif +#endif diff --git a/lib/pybind11/include/pybind11/detail/struct_smart_holder.h b/lib/pybind11/include/pybind11/detail/struct_smart_holder.h new file mode 100644 index 0000000..5d7c31c --- /dev/null +++ b/lib/pybind11/include/pybind11/detail/struct_smart_holder.h @@ -0,0 +1,398 @@ +// Copyright (c) 2020-2024 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +/* Proof-of-Concept for smart pointer interoperability. + +High-level aspects: + +* Support all `unique_ptr`, `shared_ptr` interops that are feasible. + +* Cleanly and clearly report all interops that are infeasible. + +* Meant to fit into a `PyObject`, as a holder for C++ objects. + +* Support a system design that makes it impossible to trigger + C++ Undefined Behavior, especially from Python. + +* Support a system design with clean runtime inheritance casting. From this + it follows that the `smart_holder` needs to be type-erased (`void*`). + +* Handling of RTTI for the type-erased held pointer is NOT implemented here. + It is the responsibility of the caller to ensure that `static_cast` + is well-formed when calling `as_*` member functions. Inheritance casting + needs to be handled in a different layer (similar to the code organization + in boost/python/object/inheritance.hpp). + +Details: + +* The "root holder" chosen here is a `shared_ptr` (named `vptr` in this + implementation). This choice is practically inevitable because `shared_ptr` + has only very limited support for inspecting and accessing its deleter. + +* If created from a raw pointer, or a `unique_ptr` without a custom deleter, + `vptr` always uses a custom deleter, to support `unique_ptr`-like disowning. + The custom deleters could be extended to included life-time management for + external objects (e.g. `PyObject`). + +* If created from an external `shared_ptr`, or a `unique_ptr` with a custom + deleter, including life-time management for external objects is infeasible. + +* By choice, the smart_holder is movable but not copyable, to keep the design + simple, and to guard against accidental copying overhead. + +* The `void_cast_raw_ptr` option is needed to make the `smart_holder` `vptr` + member invisible to the `shared_from_this` mechanism, in case the lifetime + of a `PyObject` is tied to the pointee. +*/ + +#pragma once + +#include "pybind11_namespace_macros.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(memory) + +// Default fallback. +static constexpr bool type_has_shared_from_this(...) { return false; } + +// This overload uses SFINAE to skip enable_shared_from_this checks when the +// base is inaccessible (e.g. private inheritance). +template +static auto type_has_shared_from_this(const T *ptr) + -> decltype(static_cast *>(ptr), true) { + return true; +} + +// Inaccessible base → substitution failure → fallback overload selected +template +static constexpr bool type_has_shared_from_this(const void *) { + return false; +} + +struct guarded_delete { + // NOTE: PYBIND11_INTERNALS_VERSION needs to be bumped if changes are made to this struct. + std::weak_ptr released_ptr; // Trick to keep the smart_holder memory footprint small. + std::function del_fun; // Rare case. + void (*del_ptr)(void *); // Common case. + bool use_del_fun; + bool armed_flag; + guarded_delete(std::function &&del_fun, bool armed_flag) + : del_fun{std::move(del_fun)}, del_ptr{nullptr}, use_del_fun{true}, + armed_flag{armed_flag} {} + guarded_delete(void (*del_ptr)(void *), bool armed_flag) + : del_ptr{del_ptr}, use_del_fun{false}, armed_flag{armed_flag} {} + void operator()(void *raw_ptr) const { + if (armed_flag) { + if (use_del_fun) { + del_fun(raw_ptr); + } else { + del_ptr(raw_ptr); + } + } + } +}; + +inline guarded_delete *get_guarded_delete(const std::shared_ptr &ptr) { + return std::get_deleter(ptr); +} + +using get_guarded_delete_fn = guarded_delete *(*) (const std::shared_ptr &); + +template ::value, int>::type = 0> +inline void std_default_delete_if_destructible(void *raw_ptr) { + std::default_delete{}(static_cast(raw_ptr)); +} + +template ::value, int>::type = 0> +inline void std_default_delete_if_destructible(void *) { + // This noop operator is needed to avoid a compilation error (for `delete raw_ptr;`), but + // throwing an exception from a destructor will std::terminate the process. Therefore the + // runtime check for lifetime-management correctness is implemented elsewhere (in + // ensure_pointee_is_destructible()). +} + +template +guarded_delete make_guarded_std_default_delete(bool armed_flag) { + return guarded_delete(std_default_delete_if_destructible, armed_flag); +} + +template +struct custom_deleter { + // NOTE: PYBIND11_INTERNALS_VERSION needs to be bumped if changes are made to this struct. + D deleter; + explicit custom_deleter(D &&deleter) : deleter{std::forward(deleter)} {} + void operator()(void *raw_ptr) { deleter(static_cast(raw_ptr)); } +}; + +template +guarded_delete make_guarded_custom_deleter(D &&uqp_del, bool armed_flag) { + return guarded_delete( + std::function(custom_deleter(std::forward(uqp_del))), armed_flag); +} + +template +constexpr bool uqp_del_is_std_default_delete() { + return std::is_same>::value + || std::is_same>::value; +} + +inline bool type_info_equal_across_dso_boundaries(const std::type_info &a, + const std::type_info &b) { + // RTTI pointer comparison may fail across DSOs (e.g., macOS libc++). + // Fallback to name comparison, which is generally safe and ABI-stable enough for our use. + return a == b || std::strcmp(a.name(), b.name()) == 0; +} + +struct smart_holder { + // NOTE: PYBIND11_INTERNALS_VERSION needs to be bumped if changes are made to this struct. + const std::type_info *rtti_uqp_del = nullptr; + std::shared_ptr vptr; + bool vptr_is_using_noop_deleter : 1; + bool vptr_is_using_std_default_delete : 1; + bool vptr_is_external_shared_ptr : 1; + bool is_populated : 1; + bool is_disowned : 1; + + // Design choice: smart_holder is movable but not copyable. + smart_holder(smart_holder &&) = default; + smart_holder(const smart_holder &) = delete; + smart_holder &operator=(smart_holder &&) = delete; + smart_holder &operator=(const smart_holder &) = delete; + + smart_holder() + : vptr_is_using_noop_deleter{false}, vptr_is_using_std_default_delete{false}, + vptr_is_external_shared_ptr{false}, is_populated{false}, is_disowned{false} {} + + bool has_pointee() const { return vptr != nullptr; } + + template + static void ensure_pointee_is_destructible(const char *context) { + if (!std::is_destructible::value) { + throw std::invalid_argument(std::string("Pointee is not destructible (") + context + + ")."); + } + } + + void ensure_is_populated(const char *context) const { + if (!is_populated) { + throw std::runtime_error(std::string("Unpopulated holder (") + context + ")."); + } + } + void ensure_is_not_disowned(const char *context) const { + if (is_disowned) { + throw std::runtime_error(std::string("Holder was disowned already (") + context + + ")."); + } + } + + void ensure_vptr_is_using_std_default_delete(const char *context) const { + if (vptr_is_external_shared_ptr) { + throw std::invalid_argument(std::string("Cannot disown external shared_ptr (") + + context + ")."); + } + if (vptr_is_using_noop_deleter) { + throw std::invalid_argument(std::string("Cannot disown non-owning holder (") + context + + ")."); + } + if (!vptr_is_using_std_default_delete) { + throw std::invalid_argument(std::string("Cannot disown custom deleter (") + context + + ")."); + } + } + + template + void ensure_compatible_uqp_del(const char *context) const { + if (!rtti_uqp_del) { + if (!uqp_del_is_std_default_delete()) { + throw std::invalid_argument(std::string("Missing unique_ptr deleter (") + context + + ")."); + } + ensure_vptr_is_using_std_default_delete(context); + return; + } + if (uqp_del_is_std_default_delete() && vptr_is_using_std_default_delete) { + return; + } + if (!type_info_equal_across_dso_boundaries(typeid(D), *rtti_uqp_del)) { + throw std::invalid_argument(std::string("Incompatible unique_ptr deleter (") + context + + ")."); + } + } + + void ensure_has_pointee(const char *context) const { + if (!has_pointee()) { + throw std::invalid_argument(std::string("Disowned holder (") + context + ")."); + } + } + + void ensure_use_count_1(const char *context) const { + if (vptr == nullptr) { + throw std::invalid_argument(std::string("Cannot disown nullptr (") + context + ")."); + } + // In multithreaded environments accessing use_count can lead to + // race conditions, but in the context of Python it is a bug (elsewhere) + // if the Global Interpreter Lock (GIL) is not being held when this code + // is reached. + // PYBIND11:REMINDER: This may need to be protected by a mutex in free-threaded Python. + if (vptr.use_count() != 1) { + throw std::invalid_argument(std::string("Cannot disown use_count != 1 (") + context + + ")."); + } + } + + void reset_vptr_deleter_armed_flag(const get_guarded_delete_fn ggd_fn, bool armed_flag) const { + auto *gd = ggd_fn(vptr); + if (gd == nullptr) { + throw std::runtime_error( + "smart_holder::reset_vptr_deleter_armed_flag() called in an invalid context."); + } + gd->armed_flag = armed_flag; + } + + // Caller is responsible for precondition: ensure_compatible_uqp_del() must succeed. + template + std::unique_ptr extract_deleter(const char *context, + const get_guarded_delete_fn ggd_fn) const { + auto *gd = ggd_fn(vptr); + if (gd && gd->use_del_fun) { + const auto &custom_deleter_ptr = gd->del_fun.template target>(); + if (custom_deleter_ptr == nullptr) { + throw std::runtime_error( + std::string("smart_holder::extract_deleter() precondition failure (") + context + + ")."); + } + static_assert(std::is_copy_constructible::value, + "Required for compatibility with smart_holder functionality."); + return std::unique_ptr(new D(custom_deleter_ptr->deleter)); + } + return nullptr; + } + + static smart_holder from_raw_ptr_unowned(void *raw_ptr) { + smart_holder hld; + hld.vptr.reset(raw_ptr, [](void *) {}); + hld.vptr_is_using_noop_deleter = true; + hld.is_populated = true; + return hld; + } + + template + T *as_raw_ptr_unowned() const { + return static_cast(vptr.get()); + } + + template + static smart_holder from_raw_ptr_take_ownership(T *raw_ptr, bool void_cast_raw_ptr = false) { + ensure_pointee_is_destructible("from_raw_ptr_take_ownership"); + smart_holder hld; + auto gd = make_guarded_std_default_delete(true); + if (void_cast_raw_ptr) { + hld.vptr.reset(static_cast(raw_ptr), std::move(gd)); + } else { + hld.vptr.reset(raw_ptr, std::move(gd)); + } + hld.vptr_is_using_std_default_delete = true; + hld.is_populated = true; + return hld; + } + + // Caller is responsible for ensuring the complex preconditions + // (see `smart_holder_type_caster_support::load_helper`). + void disown(const get_guarded_delete_fn ggd_fn) { + reset_vptr_deleter_armed_flag(ggd_fn, false); + is_disowned = true; + } + + // Caller is responsible for ensuring the complex preconditions + // (see `smart_holder_type_caster_support::load_helper`). + void reclaim_disowned(const get_guarded_delete_fn ggd_fn) { + reset_vptr_deleter_armed_flag(ggd_fn, true); + is_disowned = false; + } + + // Caller is responsible for ensuring the complex preconditions + // (see `smart_holder_type_caster_support::load_helper`). + void release_disowned() { vptr.reset(); } + + void ensure_can_release_ownership(const char *context = "ensure_can_release_ownership") const { + ensure_is_not_disowned(context); + ensure_vptr_is_using_std_default_delete(context); + ensure_use_count_1(context); + } + + // Caller is responsible for ensuring the complex preconditions + // (see `smart_holder_type_caster_support::load_helper`). + void release_ownership(const get_guarded_delete_fn ggd_fn) { + reset_vptr_deleter_armed_flag(ggd_fn, false); + release_disowned(); + } + + template + static smart_holder from_unique_ptr(std::unique_ptr &&unq_ptr, + void *mi_subobject_ptr = nullptr) { + smart_holder hld; + hld.rtti_uqp_del = &typeid(D); + hld.vptr_is_using_std_default_delete = uqp_del_is_std_default_delete(); + + // Build the owning control block on the *real object start* (T*). + guarded_delete gd + = hld.vptr_is_using_std_default_delete + ? make_guarded_std_default_delete(true) + : make_guarded_custom_deleter(std::move(unq_ptr.get_deleter()), true); + // Critical: construct owner with pointer we intend to delete + std::shared_ptr owner(unq_ptr.get(), std::move(gd)); + // Relinquish ownership only after successful construction of owner + (void) unq_ptr.release(); + + // Publish either the MI/VI subobject pointer (if provided) or the full object. + // Why this is needed: + // * The `owner` shared_ptr must always manage the true object start (T*). + // That ensures the deleter is invoked on a valid object header, so the + // virtual destructor can dispatch safely (critical on MSVC with virtual + // inheritance, where base subobjects are not at offset 0). + // * However, pybind11 needs to *register* and expose the subobject pointer + // appropriate for the type being bound. + // This pointer may differ from the T* object start under multiple/virtual + // inheritance. + // This is achieved by using an aliasing shared_ptr: + // - `owner` retains lifetime of the actual T* object start for deletion. + // - `vptr` points at the adjusted subobject (mi_subobject_ptr), giving + // Python the correct identity/registration address. + // If no subobject pointer is passed, we simply publish the full object. + if (mi_subobject_ptr) { + hld.vptr = std::shared_ptr(owner, mi_subobject_ptr); + } else { + hld.vptr = std::static_pointer_cast(owner); + } + + hld.is_populated = true; + return hld; + } + + template + static smart_holder from_shared_ptr(const std::shared_ptr &shd_ptr) { + smart_holder hld; + hld.vptr = std::static_pointer_cast(shd_ptr); + hld.vptr_is_external_shared_ptr = true; + hld.is_populated = true; + return hld; + } + + template + std::shared_ptr as_shared_ptr() const { + return std::static_pointer_cast(vptr); + } +}; + +PYBIND11_NAMESPACE_END(memory) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/detail/type_caster_base.h b/lib/pybind11/include/pybind11/detail/type_caster_base.h index 0b710d7..161b988 100644 --- a/lib/pybind11/include/pybind11/detail/type_caster_base.h +++ b/lib/pybind11/include/pybind11/detail/type_caster_base.h @@ -9,15 +9,24 @@ #pragma once -#include "../pytypes.h" +#include +#include +#include + #include "common.h" +#include "cpp_conduit.h" #include "descr.h" +#include "dynamic_raw_ptr_cast_if_possible.h" #include "internals.h" #include "typeid.h" +#include "using_smart_holder.h" +#include "value_and_holder.h" #include +#include #include #include +#include #include #include #include @@ -33,54 +42,65 @@ PYBIND11_NAMESPACE_BEGIN(detail) /// Adding a patient will keep it alive up until the enclosing function returns. class loader_life_support { private: + // Thread-local top-of-stack for loader_life_support frames (linked via parent). + // Observation: loader_life_support needs to be thread-local, + // but we don't need to go to extra effort to keep it + // per-interpreter (i.e., by putting it in internals) since + // individual function calls are already isolated to a single + // interpreter, even though they could potentially call into a + // different interpreter later in the same call chain. This + // saves a significant cost per function call spent in + // loader_life_support destruction. + // Note for future C++17 simplification: + // inline static thread_local loader_life_support *tls_current_frame = nullptr; + static loader_life_support *&tls_current_frame() { + static thread_local loader_life_support *frame_ptr = nullptr; + return frame_ptr; + } + loader_life_support *parent = nullptr; std::unordered_set keep_alive; -#if defined(WITH_THREAD) - // Store stack pointer in thread-local storage. - static PYBIND11_TLS_KEY_REF get_stack_tls_key() { -# if PYBIND11_INTERNALS_VERSION == 4 - return get_local_internals().loader_life_support_tls_key; -# else - return get_internals().loader_life_support_tls_key; -# endif - } - static loader_life_support *get_stack_top() { - return static_cast(PYBIND11_TLS_GET_VALUE(get_stack_tls_key())); - } - static void set_stack_top(loader_life_support *value) { - PYBIND11_TLS_REPLACE_VALUE(get_stack_tls_key(), value); - } -#else - // Use single global variable for stack. - static loader_life_support **get_stack_pp() { - static loader_life_support *global_stack = nullptr; - return global_stack; - } - static loader_life_support *get_stack_top() { return *get_stack_pp(); } - static void set_stack_top(loader_life_support *value) { *get_stack_pp() = value; } -#endif - public: /// A new patient frame is created when a function is entered - loader_life_support() : parent{get_stack_top()} { set_stack_top(this); } + loader_life_support() { + auto &frame = tls_current_frame(); + parent = frame; + frame = this; + } /// ... and destroyed after it returns ~loader_life_support() { - if (get_stack_top() != this) { + auto &frame = tls_current_frame(); + if (frame != this) { pybind11_fail("loader_life_support: internal error"); } - set_stack_top(parent); + frame = parent; for (auto *item : keep_alive) { Py_DECREF(item); } } + /// Keep `h` alive until the current patient frame is destroyed, if there is one. + /// Returns false when called outside a bound function (no frame). Use this, rather + /// than `add_patient`, when failing to register is acceptable because the caller + /// owns the source's lifetime outside the call framework (e.g. a view that points + /// into an existing Python object, as opposed to a freshly created temporary). + PYBIND11_NOINLINE static bool try_add_patient(handle h) { + loader_life_support *frame = tls_current_frame(); + if (!frame) { + return false; + } + if (frame->keep_alive.insert(h.ptr()).second) { + Py_INCREF(h.ptr()); + } + return true; + } + /// This can only be used inside a pybind11-bound function, either by `argument_loader` /// at argument preparation time or by `py::cast()` at execution time. PYBIND11_NOINLINE static void add_patient(handle h) { - loader_life_support *frame = get_stack_top(); - if (!frame) { + if (!try_add_patient(h)) { // NOTE: It would be nice to include the stack frames here, as this indicates // use of pybind11::cast<> outside the normal call framework, finding such // a location is challenging. Developers could consider printing out @@ -89,10 +109,6 @@ class loader_life_support { "do Python -> C++ conversions which require the creation " "of temporary values"); } - - if (frame->keep_alive.insert(h.ptr()).second) { - Py_INCREF(h.ptr()); - } } }; @@ -102,13 +118,26 @@ class loader_life_support { inline std::pair all_type_info_get_cache(PyTypeObject *type); +// Band-aid workaround to fix a subtle but serious bug in a minimalistic fashion. See PR #4762. +inline void all_type_info_add_base_most_derived_first(std::vector &bases, + type_info *addl_base) { + for (auto it = bases.begin(); it != bases.end(); it++) { + type_info *existing_base = *it; + if (PyType_IsSubtype(addl_base->type, existing_base->type) != 0) { + bases.insert(it, addl_base); + return; + } + } + bases.push_back(addl_base); +} + // Populates a just-created cache entry. PYBIND11_NOINLINE void all_type_info_populate(PyTypeObject *t, std::vector &bases) { + assert(bases.empty()); std::vector check; for (handle parent : reinterpret_borrow(t->tp_bases)) { - check.push_back((PyTypeObject *) parent.ptr()); + check.push_back(reinterpret_cast(parent.ptr())); } - auto const &type_dict = get_internals().registered_types_py; for (size_t i = 0; i < check.size(); i++) { auto *type = check[i]; @@ -136,7 +165,7 @@ PYBIND11_NOINLINE void all_type_info_populate(PyTypeObject *t, std::vectortp_bases) { @@ -150,7 +179,7 @@ PYBIND11_NOINLINE void all_type_info_populate(PyTypeObject *t, std::vector(type->tp_bases)) { - check.push_back((PyTypeObject *) parent.ptr()); + check.push_back(reinterpret_cast(parent.ptr())); } } } @@ -167,13 +196,7 @@ PYBIND11_NOINLINE void all_type_info_populate(PyTypeObject *t, std::vector &all_type_info(PyTypeObject *type) { - auto ins = all_type_info_get_cache(type); - if (ins.second) { - // New cache entry: populate it - all_type_info_populate(type, ins.first->second); - } - - return ins.first->second; + return all_type_info_get_cache(type).first->second; } /** @@ -193,32 +216,73 @@ PYBIND11_NOINLINE detail::type_info *get_type_info(PyTypeObject *type) { return bases.front(); } -inline detail::type_info *get_local_type_info(const std::type_index &tp) { - auto &locals = get_local_internals().registered_types_cpp; - auto it = locals.find(tp); +inline detail::type_info *get_local_type_info_lock_held(const std::type_info &tp) { + const auto &locals = get_local_internals().registered_types_cpp; + auto it = locals.find(&tp); if (it != locals.end()) { return it->second; } return nullptr; } -inline detail::type_info *get_global_type_info(const std::type_index &tp) { - auto &types = get_internals().registered_types_cpp; - auto it = types.find(tp); +inline detail::type_info *get_local_type_info(const std::type_info &tp) { + // NB: internals and local_internals share a single mutex + PYBIND11_LOCK_INTERNALS(get_internals()); + return get_local_type_info_lock_held(tp); +} + +inline detail::type_info *get_global_type_info_lock_held(const std::type_info &tp) { + // This is a two-level lookup. Hopefully we find the type info in + // registered_types_cpp_fast, but if not we try + // registered_types_cpp and fill registered_types_cpp_fast for + // next time. + detail::type_info *type_info = nullptr; + auto &internals = get_internals(); +#if PYBIND11_INTERNALS_VERSION >= 12 + auto &fast_types = internals.registered_types_cpp_fast; +#endif + auto &types = internals.registered_types_cpp; +#if PYBIND11_INTERNALS_VERSION >= 12 + auto fast_it = fast_types.find(&tp); + if (fast_it != fast_types.end()) { +# ifndef NDEBUG + auto types_it = types.find(std::type_index(tp)); + assert(types_it != types.end()); + assert(types_it->second == fast_it->second); +# endif + return fast_it->second; + } +#endif // PYBIND11_INTERNALS_VERSION >= 12 + + auto it = types.find(std::type_index(tp)); if (it != types.end()) { - return it->second; +#if PYBIND11_INTERNALS_VERSION >= 12 + // We found the type in the slow map but not the fast one, so + // some other DSO added it (otherwise it would be in the fast + // map under &tp) and therefore we must be an alias. Record + // that. + it->second->alias_chain.push_front(&tp); + fast_types.emplace(&tp, it->second); +#endif + type_info = it->second; } - return nullptr; + return type_info; +} + +inline detail::type_info *get_global_type_info(const std::type_info &tp) { + PYBIND11_LOCK_INTERNALS(get_internals()); + return get_global_type_info_lock_held(tp); } /// Return the type info for a given C++ type; on lookup failure can either throw or return /// nullptr. -PYBIND11_NOINLINE detail::type_info *get_type_info(const std::type_index &tp, +PYBIND11_NOINLINE detail::type_info *get_type_info(const std::type_info &tp, bool throw_if_missing = false) { - if (auto *ltype = get_local_type_info(tp)) { + PYBIND11_LOCK_INTERNALS(get_internals()); + if (auto *ltype = get_local_type_info_lock_held(tp)) { return ltype; } - if (auto *gtype = get_global_type_info(tp)) { + if (auto *gtype = get_global_type_info_lock_held(tp)) { return gtype; } @@ -233,83 +297,70 @@ PYBIND11_NOINLINE detail::type_info *get_type_info(const std::type_index &tp, PYBIND11_NOINLINE handle get_type_handle(const std::type_info &tp, bool throw_if_missing) { detail::type_info *type_info = get_type_info(tp, throw_if_missing); - return handle(type_info ? ((PyObject *) type_info->type) : nullptr); + return handle(type_info ? (reinterpret_cast(type_info->type)) : nullptr); } -// Searches the inheritance graph for a registered Python instance, using all_type_info(). -PYBIND11_NOINLINE handle find_registered_python_instance(void *src, - const detail::type_info *tinfo) { - auto it_instances = get_internals().registered_instances.equal_range(src); - for (auto it_i = it_instances.first; it_i != it_instances.second; ++it_i) { - for (auto *instance_type : detail::all_type_info(Py_TYPE(it_i->second))) { - if (instance_type && same_type(*instance_type->cpptype, *tinfo->cpptype)) { - return handle((PyObject *) it_i->second).inc_ref(); - } - } +inline bool try_incref(PyObject *obj) { + // Tries to increment the reference count of an object if it's not zero. +#if defined(Py_GIL_DISABLED) && PY_VERSION_HEX >= 0x030E00A4 + return PyUnstable_TryIncRef(obj); +#elif defined(Py_GIL_DISABLED) + // See + // https://github.com/python/cpython/blob/d05140f9f77d7dfc753dd1e5ac3a5962aaa03eff/Include/internal/pycore_object.h#L761 + uint32_t local = _Py_atomic_load_uint32_relaxed(&obj->ob_ref_local); + local += 1; + if (local == 0) { + // immortal + return true; } - return handle(); -} - -struct value_and_holder { - instance *inst = nullptr; - size_t index = 0u; - const detail::type_info *type = nullptr; - void **vh = nullptr; - - // Main constructor for a found value/holder: - value_and_holder(instance *i, const detail::type_info *type, size_t vpos, size_t index) - : inst{i}, index{index}, type{type}, vh{inst->simple_layout - ? inst->simple_value_holder - : &inst->nonsimple.values_and_holders[vpos]} {} - - // Default constructor (used to signal a value-and-holder not found by get_value_and_holder()) - value_and_holder() = default; - - // Used for past-the-end iterator - explicit value_and_holder(size_t index) : index{index} {} - - template - V *&value_ptr() const { - return reinterpret_cast(vh[0]); + if (_Py_IsOwnedByCurrentThread(obj)) { + _Py_atomic_store_uint32_relaxed(&obj->ob_ref_local, local); +# ifdef Py_REF_DEBUG + _Py_INCREF_IncRefTotal(); +# endif + return true; } - // True if this `value_and_holder` has a non-null value pointer - explicit operator bool() const { return value_ptr() != nullptr; } + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&obj->ob_ref_shared); + for (;;) { + // If the shared refcount is zero and the object is either merged + // or may not have weak references, then we cannot incref it. + if (shared == 0 || shared == _Py_REF_MERGED) { + return false; + } - template - H &holder() const { - return reinterpret_cast(vh[1]); - } - bool holder_constructed() const { - return inst->simple_layout - ? inst->simple_holder_constructed - : (inst->nonsimple.status[index] & instance::status_holder_constructed) != 0u; - } - // NOLINTNEXTLINE(readability-make-member-function-const) - void set_holder_constructed(bool v = true) { - if (inst->simple_layout) { - inst->simple_holder_constructed = v; - } else if (v) { - inst->nonsimple.status[index] |= instance::status_holder_constructed; - } else { - inst->nonsimple.status[index] &= (std::uint8_t) ~instance::status_holder_constructed; + if (_Py_atomic_compare_exchange_ssize( + &obj->ob_ref_shared, &shared, shared + (1 << _Py_REF_SHARED_SHIFT))) { +# ifdef Py_REF_DEBUG + _Py_INCREF_IncRefTotal(); +# endif + return true; } } - bool instance_registered() const { - return inst->simple_layout - ? inst->simple_instance_registered - : ((inst->nonsimple.status[index] & instance::status_instance_registered) != 0); - } - // NOLINTNEXTLINE(readability-make-member-function-const) - void set_instance_registered(bool v = true) { - if (inst->simple_layout) { - inst->simple_instance_registered = v; - } else if (v) { - inst->nonsimple.status[index] |= instance::status_instance_registered; - } else { - inst->nonsimple.status[index] &= (std::uint8_t) ~instance::status_instance_registered; +#else + assert(Py_REFCNT(obj) > 0); + Py_INCREF(obj); + return true; +#endif +} + +// Searches the inheritance graph for a registered Python instance, using all_type_info(). +PYBIND11_NOINLINE handle find_registered_python_instance(void *src, + const detail::type_info *tinfo) { + return with_instance_map(src, [&](instance_map &instances) { + auto it_instances = instances.equal_range(src); + for (auto it_i = it_instances.first; it_i != it_instances.second; ++it_i) { + for (auto *instance_type : detail::all_type_info(Py_TYPE(it_i->second))) { + if (instance_type && same_type(*instance_type->cpptype, *tinfo->cpptype)) { + auto *wrapper = reinterpret_cast(it_i->second); + if (try_incref(wrapper)) { + return handle(wrapper); + } + } + } } - } -}; + return handle(); + }); +} // Container for accessing and iterating over an instance's values/holders struct values_and_holders { @@ -322,18 +373,29 @@ struct values_and_holders { explicit values_and_holders(instance *inst) : inst{inst}, tinfo(all_type_info(Py_TYPE(inst))) {} + explicit values_and_holders(PyObject *obj) + : inst{nullptr}, tinfo(all_type_info(Py_TYPE(obj))) { + if (!tinfo.empty()) { + inst = reinterpret_cast(obj); + } + } + struct iterator { private: instance *inst = nullptr; const type_vec *types = nullptr; value_and_holder curr; friend struct values_and_holders; - iterator(instance *inst, const type_vec *tinfo) - : inst{inst}, types{tinfo}, - curr(inst /* instance */, - types->empty() ? nullptr : (*types)[0] /* type info */, - 0, /* vpos: (non-simple types only): the first vptr comes first */ - 0 /* index */) {} + iterator(instance *inst, const type_vec *tinfo) : inst{inst}, types{tinfo} { + if (inst != nullptr) { + assert(!types->empty()); + curr = value_and_holder( + inst /* instance */, + (*types)[0] /* type info */, + 0, /* vpos: (non-simple types only): the first vptr comes first */ + 0 /* index */); + } + } // Past-the-end iterator: explicit iterator(size_t end) : curr(end) {} @@ -364,6 +426,16 @@ struct values_and_holders { } size_t size() { return tinfo.size(); } + + // Band-aid workaround to fix a subtle but serious bug in a minimalistic fashion. See PR #4762. + bool is_redundant_value_and_holder(const value_and_holder &vh) { + for (size_t i = 0; i < vh.index; i++) { + if (PyType_IsSubtype(tinfo[i]->type, tinfo[vh.index]->type) != 0) { + return true; + } + } + return false; + } }; /** @@ -445,7 +517,7 @@ PYBIND11_NOINLINE void instance::allocate_layout() { // efficient for small allocations like the one we're doing here; // for larger allocations they are just wrappers around malloc. // TODO: is this still true for pure Python 3.6? - nonsimple.values_and_holders = (void **) PyMem_Calloc(space, sizeof(void *)); + nonsimple.values_and_holders = static_cast(PyMem_Calloc(space, sizeof(void *))); if (!nonsimple.values_and_holders) { throw std::bad_alloc(); } @@ -458,7 +530,7 @@ PYBIND11_NOINLINE void instance::allocate_layout() { // NOLINTNEXTLINE(readability-make-member-function-const) PYBIND11_NOINLINE void instance::deallocate_layout() { if (!simple_layout) { - PyMem_Free(nonsimple.values_and_holders); + PyMem_Free(reinterpret_cast(nonsimple.values_and_holders)); } } @@ -471,30 +543,457 @@ PYBIND11_NOINLINE bool isinstance_generic(handle obj, const std::type_info &tp) } PYBIND11_NOINLINE handle get_object_handle(const void *ptr, const detail::type_info *type) { - auto &instances = get_internals().registered_instances; - auto range = instances.equal_range(ptr); - for (auto it = range.first; it != range.second; ++it) { - for (const auto &vh : values_and_holders(it->second)) { - if (vh.type == type) { - return handle((PyObject *) it->second); + return with_instance_map(ptr, [&](instance_map &instances) { + auto range = instances.equal_range(ptr); + for (auto it = range.first; it != range.second; ++it) { + for (const auto &vh : values_and_holders(it->second)) { + if (vh.type == type) { + return handle(reinterpret_cast(it->second)); + } } } - } - return handle(); + return handle(); + }); } -inline PyThreadState *get_thread_state_unchecked() { -#if defined(PYPY_VERSION) - return PyThreadState_GET(); -#else - return _PyThreadState_UncheckedGet(); -#endif -} +// Information about how type_caster_generic::cast() can obtain its source object +struct cast_sources { + // A type-erased pointer and the type it points to + struct raw_source { + const void *cppobj; + const std::type_info *cpptype; + }; + + // A C++ pointer and the Python type info we will convert it to; + // we expect that cppobj points to something of type tinfo->cpptype + struct resolved_source { + const void *cppobj; + const type_info *tinfo; + }; + + // Use the given pointer with its compile-time type, possibly downcast + // via polymorphic_type_hook() + template + explicit cast_sources(const itype *ptr); + + // Use the given pointer and type + // NOLINTNEXTLINE(google-explicit-constructor) + cast_sources(const raw_source &orig) : original(orig) { result = resolve(); } + + // Use the given object and pybind11 type info. NB: if tinfo is null, + // this does not provide enough information to use a foreign type or + // to render a useful error message + cast_sources(const void *obj, const detail::type_info *tinfo) + : original{obj, tinfo ? tinfo->cpptype : nullptr}, result{obj, tinfo} {} + + // The object passed to cast(), with its static type. + // original.type must not be null if resolve() will be called. + // original.obj may be null if we're converting nullptr to a Python None + raw_source original; + + // A more-derived version of `original` provided by a + // polymorphic_type_hook. downcast.type may be null if this is not + // a relevant concept for the current cast. + raw_source downcast{}; + + // The source to use for this cast, and the corresponding pybind11 + // type_info. If the type_info is null, then pybind11 doesn't know + // about this type. + resolved_source result; + + // Returns true if the cast will use a pybind11 type that uses + // a smart holder. + bool creates_smart_holder() const { + return result.tinfo != nullptr + && result.tinfo->holder_enum_v == detail::holder_enum_t::smart_holder; + } + +private: + resolved_source resolve() { + if (downcast.cpptype) { + if (same_type(*original.cpptype, *downcast.cpptype)) { + downcast.cpptype = nullptr; + } else if (const auto *tpi = get_type_info(*downcast.cpptype)) { + return {downcast.cppobj, tpi}; + } + } + if (const auto *tpi = get_type_info(*original.cpptype)) { + return {original.cppobj, tpi}; + } + return {nullptr, nullptr}; + } +}; // Forward declarations void keep_alive_impl(handle nurse, handle patient); inline PyObject *make_new_instance(PyTypeObject *type); +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_GCC("-Wredundant-decls") + +// PYBIND11:REMINDER: Needs refactoring of existing pybind11 code. +inline bool deregister_instance(instance *self, void *valptr, const type_info *tinfo); + +PYBIND11_WARNING_POP + +PYBIND11_NAMESPACE_BEGIN(smart_holder_type_caster_support) + +struct value_and_holder_helper { + value_and_holder loaded_v_h; + + bool have_holder() const { + return loaded_v_h.vh != nullptr && loaded_v_h.holder_constructed(); + } + + smart_holder &holder() const { return loaded_v_h.holder(); } + + void throw_if_uninitialized_or_disowned_holder(const char *typeid_name) const { + static const std::string missing_value_msg = "Missing value for wrapped C++ type `"; + if (!holder().is_populated) { + throw value_error(missing_value_msg + clean_type_id(typeid_name) + + "`: Python instance is uninitialized."); + } + if (!holder().has_pointee()) { + throw value_error(missing_value_msg + clean_type_id(typeid_name) + + "`: Python instance was disowned."); + } + } + + void throw_if_uninitialized_or_disowned_holder(const std::type_info &type_info) const { + throw_if_uninitialized_or_disowned_holder(type_info.name()); + } + + // have_holder() must be true or this function will fail. + void throw_if_instance_is_currently_owned_by_shared_ptr(const type_info *tinfo) const { + auto *vptr_gd_ptr = tinfo->get_memory_guarded_delete(holder().vptr); + if (vptr_gd_ptr != nullptr && !vptr_gd_ptr->released_ptr.expired()) { + throw value_error("Python instance is currently owned by a std::shared_ptr."); + } + } + + void *get_void_ptr_or_nullptr() const { + if (have_holder()) { + auto &hld = holder(); + if (hld.is_populated && hld.has_pointee()) { + return hld.template as_raw_ptr_unowned(); + } + } + return nullptr; + } +}; + +template +handle smart_holder_from_unique_ptr(std::unique_ptr &&src, + return_value_policy policy, + handle parent, + const cast_sources::resolved_source &cs) { + if (policy == return_value_policy::copy) { + throw cast_error("return_value_policy::copy is invalid for unique_ptr."); + } + if (!src) { + return none().release(); + } + // cs.cppobj is the subobject pointer appropriate for tinfo (may differ from src.get() + // under MI/VI). Use this for Python identity/registration, but keep ownership on T*. + void *src_raw_void_ptr = const_cast(cs.cppobj); + assert(cs.tinfo != nullptr); + const detail::type_info *tinfo = cs.tinfo; + if (handle existing_inst = find_registered_python_instance(src_raw_void_ptr, tinfo)) { + auto *self_life_support = tinfo->get_trampoline_self_life_support(src.get()); + if (self_life_support != nullptr) { + value_and_holder &v_h = self_life_support->v_h; + if (v_h.inst != nullptr && v_h.vh != nullptr) { + auto &holder = v_h.holder(); + if (!holder.is_disowned) { + pybind11_fail("smart_holder_from_unique_ptr: unexpected " + "smart_holder.is_disowned failure."); + } + // Critical transfer-of-ownership section. This must stay together. + self_life_support->deactivate_life_support(); + holder.reclaim_disowned(tinfo->get_memory_guarded_delete); + (void) src.release(); + // Critical section end. + return existing_inst; + } + } + throw cast_error("Invalid unique_ptr: another instance owns this pointer already."); + } + + auto inst = reinterpret_steal(make_new_instance(tinfo->type)); + auto *inst_raw_ptr = reinterpret_cast(inst.ptr()); + inst_raw_ptr->owned = true; + void *&valueptr = values_and_holders(inst_raw_ptr).begin()->value_ptr(); + valueptr = src_raw_void_ptr; + + if (static_cast(src.get()) == src_raw_void_ptr) { + // This is a multiple-inheritance situation that is incompatible with the current + // shared_from_this handling (see PR #3023). Is there a better solution? + src_raw_void_ptr = nullptr; + } + auto smhldr = smart_holder::from_unique_ptr(std::move(src), src_raw_void_ptr); + tinfo->init_instance(inst_raw_ptr, static_cast(&smhldr)); + + if (policy == return_value_policy::reference_internal) { + keep_alive_impl(inst, parent); + } + + return inst.release(); +} + +template +handle smart_holder_from_unique_ptr(std::unique_ptr &&src, + return_value_policy policy, + handle parent, + const cast_sources::resolved_source &cs) { + return smart_holder_from_unique_ptr( + std::unique_ptr(const_cast(src.release()), + std::move(src.get_deleter())), // Const2Mutbl + policy, + parent, + cs); +} + +template +handle smart_holder_from_shared_ptr(const std::shared_ptr &src, + return_value_policy policy, + handle parent, + const cast_sources::resolved_source &cs) { + switch (policy) { + case return_value_policy::automatic: + case return_value_policy::automatic_reference: + break; + case return_value_policy::take_ownership: + throw cast_error("Invalid return_value_policy for shared_ptr (take_ownership)."); + case return_value_policy::copy: + case return_value_policy::move: + break; + case return_value_policy::reference: + throw cast_error("Invalid return_value_policy for shared_ptr (reference)."); + case return_value_policy::reference_internal: + break; + } + if (!src) { + return none().release(); + } + + // cs.cppobj is the subobject pointer appropriate for tinfo (may differ from src.get() + // under MI/VI). Use this for Python identity/registration, but keep ownership on T*. + void *src_raw_void_ptr = const_cast(cs.cppobj); + assert(cs.tinfo != nullptr); + const detail::type_info *tinfo = cs.tinfo; + if (handle existing_inst = find_registered_python_instance(src_raw_void_ptr, tinfo)) { + // PYBIND11:REMINDER: MISSING: Enforcement of consistency with existing smart_holder. + // PYBIND11:REMINDER: MISSING: keep_alive. + return existing_inst; + } + + auto inst = reinterpret_steal(make_new_instance(tinfo->type)); + auto *inst_raw_ptr = reinterpret_cast(inst.ptr()); + inst_raw_ptr->owned = true; + void *&valueptr = values_and_holders(inst_raw_ptr).begin()->value_ptr(); + valueptr = src_raw_void_ptr; + + auto smhldr = smart_holder::from_shared_ptr(std::shared_ptr(src, src_raw_void_ptr)); + tinfo->init_instance(inst_raw_ptr, static_cast(&smhldr)); + + if (policy == return_value_policy::reference_internal) { + keep_alive_impl(inst, parent); + } + + return inst.release(); +} + +template +handle smart_holder_from_shared_ptr(const std::shared_ptr &src, + return_value_policy policy, + handle parent, + const cast_sources::resolved_source &cs) { + return smart_holder_from_shared_ptr(std::const_pointer_cast(src), // Const2Mutbl + policy, + parent, + cs); +} + +struct shared_ptr_parent_life_support { + PyObject *parent; + explicit shared_ptr_parent_life_support(PyObject *parent) : parent{parent} { + Py_INCREF(parent); + } + // NOLINTNEXTLINE(readability-make-member-function-const) + void operator()(void *) { + gil_scoped_acquire gil; + Py_DECREF(parent); + } +}; + +struct shared_ptr_trampoline_self_life_support { + PyObject *self; + explicit shared_ptr_trampoline_self_life_support(instance *inst) + : self{reinterpret_cast(inst)} { + gil_scoped_acquire gil; + Py_INCREF(self); + } + // NOLINTNEXTLINE(readability-make-member-function-const) + void operator()(void *) { + gil_scoped_acquire gil; + Py_DECREF(self); + } +}; + +template ::value, int>::type = 0> +inline std::unique_ptr unique_with_deleter(T *raw_ptr, std::unique_ptr &&deleter) { + if (deleter == nullptr) { + return std::unique_ptr(raw_ptr); + } + return std::unique_ptr(raw_ptr, std::move(*deleter)); +} + +template ::value, int>::type = 0> +inline std::unique_ptr unique_with_deleter(T *raw_ptr, std::unique_ptr &&deleter) { + if (deleter == nullptr) { + pybind11_fail("smart_holder_type_casters: deleter is not default constructible and no" + " instance available to return."); + } + return std::unique_ptr(raw_ptr, std::move(*deleter)); +} + +template +struct load_helper : value_and_holder_helper { + bool was_populated = false; + bool python_instance_is_alias = false; + + void maybe_set_python_instance_is_alias(handle src) { + if (was_populated) { + python_instance_is_alias = reinterpret_cast(src.ptr())->is_alias; + } + } + + static std::shared_ptr make_shared_ptr_with_responsible_parent(T *raw_ptr, handle parent) { + return std::shared_ptr(raw_ptr, shared_ptr_parent_life_support(parent.ptr())); + } + + std::shared_ptr load_as_shared_ptr(const type_info *tinfo, + void *void_raw_ptr, + handle responsible_parent = nullptr, + // to support py::potentially_slicing_weak_ptr + // with minimal added code complexity: + bool force_potentially_slicing_shared_ptr + = false) const { + if (!have_holder()) { + return nullptr; + } + throw_if_uninitialized_or_disowned_holder(typeid(T)); + smart_holder &hld = holder(); + hld.ensure_is_not_disowned("load_as_shared_ptr"); + if (hld.vptr_is_using_noop_deleter) { + if (responsible_parent) { + return make_shared_ptr_with_responsible_parent(static_cast(void_raw_ptr), + responsible_parent); + } + throw std::runtime_error("Non-owning holder (load_as_shared_ptr)."); + } + auto *type_raw_ptr = static_cast(void_raw_ptr); + if (python_instance_is_alias && !force_potentially_slicing_shared_ptr) { + auto *vptr_gd_ptr = tinfo->get_memory_guarded_delete(holder().vptr); + if (vptr_gd_ptr != nullptr) { + std::shared_ptr released_ptr = vptr_gd_ptr->released_ptr.lock(); + if (released_ptr) { + return std::shared_ptr(released_ptr, type_raw_ptr); + } + std::shared_ptr to_be_released( + type_raw_ptr, shared_ptr_trampoline_self_life_support(loaded_v_h.inst)); + vptr_gd_ptr->released_ptr = to_be_released; + return to_be_released; + } + auto *sptsls_ptr = std::get_deleter(hld.vptr); + if (sptsls_ptr != nullptr) { + // This code is reachable only if there are multiple registered_instances for the + // same pointee. + if (reinterpret_cast(loaded_v_h.inst) == sptsls_ptr->self) { + pybind11_fail("smart_holder_type_caster_support load_as_shared_ptr failure: " + "loaded_v_h.inst == sptsls_ptr->self"); + } + } + if (sptsls_ptr != nullptr || !memory::type_has_shared_from_this(type_raw_ptr)) { + return std::shared_ptr( + type_raw_ptr, shared_ptr_trampoline_self_life_support(loaded_v_h.inst)); + } + if (hld.vptr_is_external_shared_ptr) { + pybind11_fail("smart_holder_type_casters load_as_shared_ptr failure: not " + "implemented: trampoline-self-life-support for external shared_ptr " + "to type inheriting from std::enable_shared_from_this."); + } + pybind11_fail( + "smart_holder_type_casters: load_as_shared_ptr failure: internal inconsistency."); + } + std::shared_ptr void_shd_ptr = hld.template as_shared_ptr(); + return std::shared_ptr(void_shd_ptr, type_raw_ptr); + } + + template + std::unique_ptr load_as_unique_ptr(const type_info *tinfo, + void *raw_void_ptr, + const char *context = "load_as_unique_ptr") { + if (!have_holder()) { + return unique_with_deleter(nullptr, std::unique_ptr()); + } + throw_if_uninitialized_or_disowned_holder(typeid(T)); + throw_if_instance_is_currently_owned_by_shared_ptr(tinfo); + holder().ensure_is_not_disowned(context); + holder().template ensure_compatible_uqp_del(context); + holder().ensure_use_count_1(context); + + T *raw_type_ptr = static_cast(raw_void_ptr); + + auto *self_life_support = tinfo->get_trampoline_self_life_support(raw_type_ptr); + // This is enforced indirectly by a static_assert in the class_ implementation: + assert(!python_instance_is_alias || self_life_support); + + std::unique_ptr extracted_deleter + = holder().template extract_deleter(context, tinfo->get_memory_guarded_delete); + + // Critical transfer-of-ownership section. This must stay together. + if (self_life_support != nullptr) { + holder().disown(tinfo->get_memory_guarded_delete); + } else { + holder().release_ownership(tinfo->get_memory_guarded_delete); + } + auto result = unique_with_deleter(raw_type_ptr, std::move(extracted_deleter)); + if (self_life_support != nullptr) { + self_life_support->activate_life_support(loaded_v_h); + } else { + void *value_void_ptr = loaded_v_h.value_ptr(); + loaded_v_h.value_ptr() = nullptr; + deregister_instance(loaded_v_h.inst, value_void_ptr, loaded_v_h.type); + } + // Critical section end. + + return result; + } + + // This assumes load_as_shared_ptr succeeded(), and the returned shared_ptr is still alive. + // The returned unique_ptr is meant to never expire (the behavior is undefined otherwise). + template + std::unique_ptr load_as_const_unique_ptr(const type_info *tinfo, + T *raw_type_ptr, + const char *context + = "load_as_const_unique_ptr") { + if (!have_holder()) { + return unique_with_deleter(nullptr, std::unique_ptr()); + } + holder().template ensure_compatible_uqp_del(context); + return unique_with_deleter(raw_type_ptr, + std::move(holder().template extract_deleter( + context, tinfo->get_memory_guarded_delete))); + } +}; + +PYBIND11_NAMESPACE_END(smart_holder_type_caster_support) + class type_caster_generic { public: PYBIND11_NOINLINE explicit type_caster_generic(const std::type_info &type_info) @@ -505,21 +1004,51 @@ class type_caster_generic { bool load(handle src, bool convert) { return load_impl(src, convert); } - PYBIND11_NOINLINE static handle cast(const void *_src, + static handle cast(const void *src, + return_value_policy policy, + handle parent, + const detail::type_info *tinfo, + void *(*copy_constructor)(const void *), + void *(*move_constructor)(const void *), + const void *existing_holder = nullptr) { + cast_sources srcs{src, tinfo}; + return cast(srcs, policy, parent, copy_constructor, move_constructor, existing_holder); + } + + static handle cast_non_owning(const cast_sources &srcs, + return_value_policy policy, + handle parent, + const void *existing_holder = nullptr) { + // Reference-like policies alias an existing C++ object instead of creating + // a new one, so copy/move constructor callbacks must remain null here. + assert(policy == return_value_policy::reference + || policy == return_value_policy::reference_internal + || policy == return_value_policy::automatic_reference); + return cast(srcs, policy, parent, nullptr, nullptr, existing_holder); + } + + PYBIND11_NOINLINE static handle cast(const cast_sources &srcs, return_value_policy policy, handle parent, - const detail::type_info *tinfo, void *(*copy_constructor)(const void *), void *(*move_constructor)(const void *), const void *existing_holder = nullptr) { - if (!tinfo) { // no type info: error will be set already + if (!srcs.result.tinfo) { + // No pybind11 type info. Raise an exception. + std::string tname = srcs.downcast.cpptype ? srcs.downcast.cpptype->name() + : srcs.original.cpptype ? srcs.original.cpptype->name() + : ""; + detail::clean_type_id(tname); + std::string msg = "Unregistered type : " + tname; + set_error(PyExc_TypeError, msg.c_str()); return handle(); } - void *src = const_cast(_src); + void *src = const_cast(srcs.result.cppobj); if (src == nullptr) { return none().release(); } + const type_info *tinfo = srcs.result.tinfo; if (handle registered_inst = find_registered_python_instance(src, tinfo)) { return registered_inst; @@ -599,6 +1128,15 @@ class type_caster_generic { // Base methods for generic caster; there are overridden in copyable_holder_caster void load_value(value_and_holder &&v_h) { + if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) { + smart_holder_type_caster_support::value_and_holder_helper v_h_helper; + v_h_helper.loaded_v_h = v_h; + if (v_h_helper.have_holder()) { + v_h_helper.throw_if_uninitialized_or_disowned_holder(cpptype->name()); + value = v_h_helper.holder().template as_raw_ptr_unowned(); + return; + } + } auto *&vptr = v_h.value_ptr(); // Lazy allocation for unallocated values: if (vptr == nullptr) { @@ -606,7 +1144,7 @@ class type_caster_generic { if (type->operator_new) { vptr = type->operator_new(type->type_size); } else { -#if defined(__cpp_aligned_new) && (!defined(_MSC_VER) || _MSC_VER >= 1912) +#if defined(__cpp_aligned_new) if (type->type_align > __STDCPP_DEFAULT_NEW_ALIGNMENT__) { vptr = ::operator new(type->type_size, std::align_val_t(type->type_align)); } else { @@ -637,7 +1175,15 @@ class type_caster_generic { } return false; } + bool try_cpp_conduit(handle src) { + value = try_raw_pointer_ephemeral_from_cpp_conduit(src, cpptype); + if (value != nullptr) { + return true; + } + return false; + } void check_holder_compat() {} + bool set_foreign_holder(handle) { return true; } PYBIND11_NOINLINE static void *local_load(PyObject *src, const type_info *ti) { auto caster = type_caster_generic(ti); @@ -676,14 +1222,14 @@ class type_caster_generic { // logic (without having to resort to virtual inheritance). template PYBIND11_NOINLINE bool load_impl(handle src, bool convert) { + auto &this_ = static_cast(*this); if (!src) { return false; } if (!typeinfo) { - return try_load_foreign_module_local(src); + return try_load_foreign_module_local(src) && this_.set_foreign_holder(src); } - auto &this_ = static_cast(*this); this_.check_holder_compat(); PyTypeObject *srctype = Py_TYPE(src.ptr()); @@ -749,13 +1295,13 @@ class type_caster_generic { if (typeinfo->module_local) { if (auto *gtype = get_global_type_info(*typeinfo->cpptype)) { typeinfo = gtype; - return load(src, false); + return load_impl(src, false); } } // Global typeinfo has precedence over foreign module_local if (try_load_foreign_module_local(src)) { - return true; + return this_.set_foreign_holder(src); } // Custom converters didn't take None, now we convert None to nullptr. @@ -768,26 +1314,11 @@ class type_caster_generic { return true; } - return false; - } - - // Called to do type lookup and wrap the pointer and type in a pair when a dynamic_cast - // isn't needed or can't be used. If the type is unknown, sets the error and returns a pair - // with .second = nullptr. (p.first = nullptr is not an error: it becomes None). - PYBIND11_NOINLINE static std::pair - src_and_type(const void *src, - const std::type_info &cast_type, - const std::type_info *rtti_type = nullptr) { - if (auto *tpi = get_type_info(cast_type)) { - return {src, const_cast(tpi)}; + if (convert && cpptype && this_.try_cpp_conduit(src)) { + return this_.set_foreign_holder(src); } - // Not found, set error: - std::string tname = rtti_type ? rtti_type->name() : cast_type.name(); - detail::clean_type_id(tname); - std::string msg = "Unregistered type : " + tname; - PyErr_SetString(PyExc_TypeError, msg.c_str()); - return {nullptr, nullptr}; + return false; } const type_info *typeinfo = nullptr; @@ -795,6 +1326,32 @@ class type_caster_generic { void *value = nullptr; }; +inline object cpp_conduit_method(handle self, + const bytes &pybind11_platform_abi_id, + const capsule &cpp_type_info_capsule, + const bytes &pointer_kind) { +#ifdef PYBIND11_HAS_STRING_VIEW + using cpp_str = std::string_view; +#else + using cpp_str = std::string; +#endif + if (cpp_str(pybind11_platform_abi_id) != PYBIND11_PLATFORM_ABI_ID) { + return none(); + } + if (std::strcmp(cpp_type_info_capsule.name(), typeid(std::type_info).name()) != 0) { + return none(); + } + if (cpp_str(pointer_kind) != "raw_pointer_ephemeral") { + throw std::runtime_error("Invalid pointer_kind: \"" + std::string(pointer_kind) + "\""); + } + const auto *cpp_type_info = cpp_type_info_capsule.get_pointer(); + type_caster_generic caster(*cpp_type_info); + if (!caster.load(self, false)) { + return none(); + } + return capsule(caster.value, cpp_type_info->name()); +} + /** * Determine suitable casting operator for pointer-or-lvalue-casting type casters. The type caster * needs to provide `operator T*()` and `operator T&()` operators. @@ -822,23 +1379,179 @@ using movable_cast_op_type typename std::add_rvalue_reference>::type, typename std::add_lvalue_reference>::type>>; -// std::is_copy_constructible isn't quite enough: it lets std::vector (and similar) through when -// T is non-copyable, but code containing such a copy constructor fails to actually compile. -template -struct is_copy_constructible : std::is_copy_constructible {}; +// Does the container have a mapped type and is it recursive? +// Implemented by specializations below. +template +struct container_mapped_type_traits { + static constexpr bool has_mapped_type = false; + static constexpr bool has_recursive_mapped_type = false; +}; + +template +struct container_mapped_type_traits< + Container, + typename std::enable_if< + std::is_same::value>::type> { + static constexpr bool has_mapped_type = true; + static constexpr bool has_recursive_mapped_type = true; +}; + +template +struct container_mapped_type_traits< + Container, + typename std::enable_if< + negation>::value>::type> { + static constexpr bool has_mapped_type = true; + static constexpr bool has_recursive_mapped_type = false; +}; + +// Does the container have a value type and is it recursive? +// Implemented by specializations below. +template +struct container_value_type_traits : std::false_type { + static constexpr bool has_value_type = false; + static constexpr bool has_recursive_value_type = false; +}; + +template +struct container_value_type_traits< + Container, + typename std::enable_if< + std::is_same::value>::type> { + static constexpr bool has_value_type = true; + static constexpr bool has_recursive_value_type = true; +}; -// Specialization for types that appear to be copy constructible but also look like stl containers -// (we specifically check for: has `value_type` and `reference` with `reference = value_type&`): if -// so, copy constructability depends on whether the value_type is copy constructible. template -struct is_copy_constructible< +struct container_value_type_traits< Container, - enable_if_t< - all_of, - std::is_same, - // Avoid infinite recursion - negation>>::value>> - : is_copy_constructible {}; + typename std::enable_if< + negation>::value>::type> { + static constexpr bool has_value_type = true; + static constexpr bool has_recursive_value_type = false; +}; + +/* + * Tag to be used for representing the bottom of recursively defined types. + * Define this tag so we don't have to use void. + */ +struct recursive_bottom {}; + +/* + * Implementation detail of `recursive_container_traits` below. + * `T` is the `value_type` of the container, which might need to be modified to + * avoid recursive types and const types. + */ +template +struct impl_type_to_check_recursively { + /* + * If the container is recursive, then no further recursion should be done. + */ + using if_recursive = recursive_bottom; + /* + * Otherwise yield `T` unchanged. + */ + using if_not_recursive = T; +}; + +/* + * For pairs - only as value type of a map -, the first type should remove the `const`. + * Also, if the map is recursive, then the recursive checking should consider + * the first type only. + */ +template +struct impl_type_to_check_recursively, /* is_this_a_map = */ true> { + using if_recursive = typename std::remove_const::type; + using if_not_recursive = std::pair::type, B>; +}; + +/* + * Implementation of `recursive_container_traits` below. + */ +template +struct impl_recursive_container_traits { + using type_to_check_recursively = recursive_bottom; +}; + +template +struct impl_recursive_container_traits< + Container, + typename std::enable_if::has_value_type>::type> { + static constexpr bool is_recursive + = container_mapped_type_traits::has_recursive_mapped_type + || container_value_type_traits::has_recursive_value_type; + /* + * This member dictates which type Pybind11 should check recursively in traits + * such as `is_move_constructible`, `is_copy_constructible`, `is_move_assignable`, ... + * Direct access to `value_type` should be avoided: + * 1. `value_type` might recursively contain the type again + * 2. `value_type` of STL map types is `std::pair`, the `const` + * should be removed. + * + */ + using type_to_check_recursively = typename std::conditional< + is_recursive, + typename impl_type_to_check_recursively< + typename Container::value_type, + container_mapped_type_traits::has_mapped_type>::if_recursive, + typename impl_type_to_check_recursively< + typename Container::value_type, + container_mapped_type_traits::has_mapped_type>::if_not_recursive>::type; +}; + +/* + * This trait defines the `type_to_check_recursively` which is needed to properly + * handle recursively defined traits such as `is_move_constructible` without going + * into an infinite recursion. + * Should be used instead of directly accessing the `value_type`. + * It cancels the recursion by returning the `recursive_bottom` tag. + * + * The default definition of `type_to_check_recursively` is as follows: + * + * 1. By default, it is `recursive_bottom`, so that the recursion is canceled. + * 2. If the type is non-recursive and defines a `value_type`, then the `value_type` is used. + * If the `value_type` is a pair and a `mapped_type` is defined, + * then the `const` is removed from the first type. + * 3. If the type is recursive and `value_type` is not a pair, then `recursive_bottom` is returned. + * 4. If the type is recursive and `value_type` is a pair and a `mapped_type` is defined, + * then `const` is removed from the first type and the first type is returned. + * + * This behavior can be extended by the user as seen in test_stl_binders.cpp. + * + * This struct is exactly the same as impl_recursive_container_traits. + * The duplication achieves that user-defined specializations don't compete + * with internal specializations, but take precedence. + */ +template +struct recursive_container_traits : impl_recursive_container_traits {}; + +template +struct is_move_constructible + : all_of, + is_move_constructible< + typename recursive_container_traits::type_to_check_recursively>> {}; + +template <> +struct is_move_constructible : std::true_type {}; + +// Likewise for std::pair +// (after C++17 it is mandatory that the move constructor not exist when the two types aren't +// themselves move constructible, but this can not be relied upon when T1 or T2 are themselves +// containers). +template +struct is_move_constructible> + : all_of, is_move_constructible> {}; + +// std::is_copy_constructible isn't quite enough: it lets std::vector (and similar) through when +// T is non-copyable, but code containing such a copy constructor fails to actually compile. +template +struct is_copy_constructible + : all_of, + is_copy_constructible< + typename recursive_container_traits::type_to_check_recursively>> {}; + +template <> +struct is_copy_constructible : std::true_type {}; // Likewise for std::pair // (after C++17 it is mandatory that the copy constructor not exist when the two types aren't @@ -849,14 +1562,16 @@ struct is_copy_constructible> : all_of, is_copy_constructible> {}; // The same problems arise with std::is_copy_assignable, so we use the same workaround. -template -struct is_copy_assignable : std::is_copy_assignable {}; -template -struct is_copy_assignable, - std::is_same>::value>> - : is_copy_assignable {}; +template +struct is_copy_assignable + : all_of< + std::is_copy_assignable, + is_copy_assignable::type_to_check_recursively>> { +}; + +template <> +struct is_copy_assignable : std::true_type {}; + template struct is_copy_assignable> : all_of, is_copy_assignable> {}; @@ -900,6 +1615,20 @@ struct polymorphic_type_hook : public polymorphic_type_hook_base {}; PYBIND11_NAMESPACE_BEGIN(detail) +template +cast_sources::cast_sources(const itype *ptr) : original{ptr, &typeid(itype)} { + // If this is a base pointer to a derived type, and the derived type is + // registered with pybind11, we want to make the full derived object + // available. In the typical case where itype is polymorphic, we get the + // correct derived pointer (which may be != base pointer) by a dynamic_cast + // to most derived type. If itype is not polymorphic, a user-provided + // specialization of polymorphic_type_hook can do the same thing. + // If there is no downcast to perform, then the default hook will leave + // derived.type set to nullptr, which causes us to ignore derived.obj. + downcast.cppobj = polymorphic_type_hook::get(ptr, downcast.cpptype); + result = resolve(); +} + /// Generic type caster for objects stored on the heap template class type_caster_base : public type_caster_generic { @@ -911,62 +1640,44 @@ class type_caster_base : public type_caster_generic { type_caster_base() : type_caster_base(typeid(type)) {} explicit type_caster_base(const std::type_info &info) : type_caster_generic(info) {} + // Wrap the generic cast_sources to be only constructible from the type + // that's correct in this context, so you can't use type_caster_base + // to convert an unrelated B* to Python. + struct cast_sources : detail::cast_sources { + explicit cast_sources(const itype *ptr) : detail::cast_sources(ptr) {} + }; + static handle cast(const itype &src, return_value_policy policy, handle parent) { if (policy == return_value_policy::automatic || policy == return_value_policy::automatic_reference) { policy = return_value_policy::copy; } - return cast(&src, policy, parent); + return cast(std::addressof(src), policy, parent); } static handle cast(itype &&src, return_value_policy, handle parent) { - return cast(&src, return_value_policy::move, parent); - } - - // Returns a (pointer, type_info) pair taking care of necessary type lookup for a - // polymorphic type (using RTTI by default, but can be overridden by specializing - // polymorphic_type_hook). If the instance isn't derived, returns the base version. - static std::pair src_and_type(const itype *src) { - const auto &cast_type = typeid(itype); - const std::type_info *instance_type = nullptr; - const void *vsrc = polymorphic_type_hook::get(src, instance_type); - if (instance_type && !same_type(cast_type, *instance_type)) { - // This is a base pointer to a derived type. If the derived type is registered - // with pybind11, we want to make the full derived object available. - // In the typical case where itype is polymorphic, we get the correct - // derived pointer (which may be != base pointer) by a dynamic_cast to - // most derived type. If itype is not polymorphic, we won't get here - // except via a user-provided specialization of polymorphic_type_hook, - // and the user has promised that no this-pointer adjustment is - // required in that case, so it's OK to use static_cast. - if (const auto *tpi = get_type_info(*instance_type)) { - return {vsrc, tpi}; - } - } - // Otherwise we have either a nullptr, an `itype` pointer, or an unknown derived pointer, - // so don't do a cast - return type_caster_generic::src_and_type(src, cast_type, instance_type); + return cast(std::addressof(src), return_value_policy::move, parent); } static handle cast(const itype *src, return_value_policy policy, handle parent) { - auto st = src_and_type(src); - return type_caster_generic::cast(st.first, + return cast(cast_sources{src}, policy, parent); + } + + static handle cast(const cast_sources &srcs, return_value_policy policy, handle parent) { + return type_caster_generic::cast(srcs, policy, parent, - st.second, - make_copy_constructor(src), - make_move_constructor(src)); + make_copy_constructor((const itype *) nullptr), + make_move_constructor((const itype *) nullptr)); } static handle cast_holder(const itype *src, const void *holder) { - auto st = src_and_type(src); - return type_caster_generic::cast(st.first, - return_value_policy::take_ownership, - {}, - st.second, - nullptr, - nullptr, - holder); + return cast_holder(cast_sources{src}, holder); + } + + static handle cast_holder(const cast_sources &srcs, const void *holder) { + auto policy = return_value_policy::take_ownership; + return type_caster_generic::cast(srcs, policy, {}, nullptr, nullptr, holder); } template @@ -994,7 +1705,7 @@ class type_caster_base : public type_caster_generic { return [](const void *arg) -> void * { return new T(*reinterpret_cast(arg)); }; } - template ::value>> + template ::value>> static auto make_move_constructor(const T *) -> decltype(new T(std::declval()), Constructor{}) { return [](const void *arg) -> void * { @@ -1006,13 +1717,17 @@ class type_caster_base : public type_caster_generic { static Constructor make_move_constructor(...) { return nullptr; } }; +inline std::string quote_cpp_type_name(const std::string &cpp_type_name) { + return cpp_type_name; // No-op for now. See PR #4888 +} + PYBIND11_NOINLINE std::string type_info_description(const std::type_info &ti) { if (auto *type_data = get_type_info(ti)) { - handle th((PyObject *) type_data->type); + handle th(reinterpret_cast(type_data->type)); return th.attr("__module__").cast() + '.' + th.attr("__qualname__").cast(); } - return clean_type_id(ti.name()); + return quote_cpp_type_name(clean_type_id(ti.name())); } PYBIND11_NAMESPACE_END(detail) diff --git a/lib/pybind11/include/pybind11/detail/using_smart_holder.h b/lib/pybind11/include/pybind11/detail/using_smart_holder.h new file mode 100644 index 0000000..9031489 --- /dev/null +++ b/lib/pybind11/include/pybind11/detail/using_smart_holder.h @@ -0,0 +1,22 @@ +// Copyright (c) 2024 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#pragma once + +#include "common.h" +#include "struct_smart_holder.h" + +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +using pybind11::memory::smart_holder; + +PYBIND11_NAMESPACE_BEGIN(detail) + +template +using is_smart_holder = std::is_same; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/detail/value_and_holder.h b/lib/pybind11/include/pybind11/detail/value_and_holder.h new file mode 100644 index 0000000..b24551e --- /dev/null +++ b/lib/pybind11/include/pybind11/detail/value_and_holder.h @@ -0,0 +1,92 @@ +// Copyright (c) 2016-2024 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#pragma once + +#include "common.h" + +#include +#include +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +struct value_and_holder { + instance *inst = nullptr; + size_t index = 0u; + const detail::type_info *type = nullptr; + void **vh = nullptr; + + // Main constructor for a found value/holder: + value_and_holder(instance *i, const detail::type_info *type, size_t vpos, size_t index) + : inst{i}, index{index}, type{type}, + vh{inst->simple_layout ? inst->simple_value_holder + : &inst->nonsimple.values_and_holders[vpos]} {} + + // Default constructor (used to signal a value-and-holder not found by get_value_and_holder()) + value_and_holder() = default; + + // Used for past-the-end iterator + explicit value_and_holder(size_t index) : index{index} {} + + template + V *&value_ptr() const { + return reinterpret_cast(vh[0]); + } + // True if this `value_and_holder` has a non-null value pointer + explicit operator bool() const { return value_ptr() != nullptr; } + + template + H &holder() const { + return reinterpret_cast(vh[1]); + } + bool holder_constructed() const { + return inst->simple_layout + ? inst->simple_holder_constructed + : (inst->nonsimple.status[index] & instance::status_holder_constructed) != 0u; + } + // NOLINTNEXTLINE(readability-make-member-function-const) + void set_holder_constructed(bool v = true) { + if (inst->simple_layout) { + inst->simple_holder_constructed = v; + } else if (v) { + inst->nonsimple.status[index] |= instance::status_holder_constructed; + } else { + inst->nonsimple.status[index] + &= static_cast(~instance::status_holder_constructed); + } + } + bool instance_registered() const { + return inst->simple_layout + ? inst->simple_instance_registered + : ((inst->nonsimple.status[index] & instance::status_instance_registered) != 0); + } + // NOLINTNEXTLINE(readability-make-member-function-const) + void set_instance_registered(bool v = true) { + if (inst->simple_layout) { + inst->simple_instance_registered = v; + } else if (v) { + inst->nonsimple.status[index] |= instance::status_instance_registered; + } else { + inst->nonsimple.status[index] + &= static_cast(~instance::status_instance_registered); + } + } +}; + +// This is a semi-public API to check if the corresponding instance has been constructed with a +// holder. That is, if the instance has been constructed with a holder, the `__init__` method is +// called and the C++ object is valid. Otherwise, the C++ object might only be allocated, but not +// initialized. This will lead to **SEGMENTATION FAULTS** if the C++ object is used in any way. +// Example usage: https://pybind11.readthedocs.io/en/stable/advanced/classes.html#custom-type-setup +// for `tp_traverse` and `tp_clear` implementations. +// WARNING: The caller is responsible for ensuring that the `reinterpret_cast` is valid. +inline bool is_holder_constructed(PyObject *obj) { + auto *const instance = reinterpret_cast(obj); + return instance->get_value_and_holder().holder_constructed(); +} + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/eigen/common.h b/lib/pybind11/include/pybind11/eigen/common.h new file mode 100644 index 0000000..24f56d1 --- /dev/null +++ b/lib/pybind11/include/pybind11/eigen/common.h @@ -0,0 +1,9 @@ +// Copyright (c) 2023 The pybind Community. + +#pragma once + +// Common message for `static_assert()`s, which are useful to easily +// preempt much less obvious errors. +#define PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED \ + "Pointer types (in particular `PyObject *`) are not supported as scalar types for Eigen " \ + "types." diff --git a/lib/pybind11/include/pybind11/eigen/matrix.h b/lib/pybind11/include/pybind11/eigen/matrix.h index 34fe329..ca599c9 100644 --- a/lib/pybind11/include/pybind11/eigen/matrix.h +++ b/lib/pybind11/include/pybind11/eigen/matrix.h @@ -9,7 +9,9 @@ #pragma once -#include "../numpy.h" +#include + +#include "common.h" /* HINT: To suppress warnings originating from the Eigen headers, use -isystem. See also: @@ -19,7 +21,9 @@ PYBIND11_WARNING_PUSH PYBIND11_WARNING_DISABLE_MSVC(5054) // https://github.com/pybind/pybind11/pull/3741 // C5054: operator '&': deprecated between enumerations of different types +#if defined(__MINGW32__) PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized") +#endif #include #include @@ -221,19 +225,22 @@ struct EigenProps { = !show_c_contiguous && show_order && requires_col_major; static constexpr auto descriptor - = const_name("numpy.ndarray[") + npy_format_descriptor::name + const_name("[") + = const_name("typing.Annotated[") + + io_name("numpy.typing.ArrayLike, ", "numpy.typing.NDArray[") + + npy_format_descriptor::name + io_name("", "]") + const_name(", \"[") + const_name(const_name<(size_t) rows>(), const_name("m")) + const_name(", ") - + const_name(const_name<(size_t) cols>(), const_name("n")) + const_name("]") - + + + const_name(const_name<(size_t) cols>(), const_name("n")) + + const_name("]\"") // For a reference type (e.g. Ref) we have other constraints that might need to // be satisfied: writeable=True (for a mutable reference), and, depending on the map's // stride options, possibly f_contiguous or c_contiguous. We include them in the // descriptor output to provide some hint as to why a TypeError is occurring (otherwise - // it can be confusing to see that a function accepts a 'numpy.ndarray[float64[3,2]]' and - // an error message that you *gave* a numpy.ndarray of the right type and dimensions. - const_name(", flags.writeable", "") - + const_name(", flags.c_contiguous", "") - + const_name(", flags.f_contiguous", "") + const_name("]"); + // it can be confusing to see that a function accepts a + // 'typing.Annotated[numpy.typing.NDArray[numpy.float64], "[3,2]"]' and an error message + // that you *gave* a numpy.ndarray of the right type and dimensions. + + const_name(", \"flags.writeable\"", "") + + const_name(", \"flags.c_contiguous\"", "") + + const_name(", \"flags.f_contiguous\"", "") + const_name("]"); }; // Casts an Eigen type to numpy array. If given a base, the numpy array references the src data, @@ -285,6 +292,8 @@ handle eigen_encapsulate(Type *src) { template struct type_caster::value>> { using Scalar = typename Type::Scalar; + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using props = EigenProps; bool load(handle src, bool convert) { @@ -310,8 +319,11 @@ struct type_caster::value>> { return false; } + PYBIND11_WARNING_PUSH + PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized") // See PR #5516 // Allocate the new type, then build a numpy reference into it value = Type(fits.rows, fits.cols); + PYBIND11_WARNING_POP auto ref = reinterpret_steal(eigen_ref_array(value)); if (dims == 1) { ref = ref.squeeze(); @@ -403,6 +415,9 @@ struct type_caster::value>> { // Base class for casting reference/map/block/etc. objects back to python. template struct eigen_map_caster { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + private: using props = EigenProps; @@ -429,7 +444,9 @@ struct eigen_map_caster { } } - static constexpr auto name = props::descriptor; + // return_descr forces the use of NDArray instead of ArrayLike in args + // since Ref<...> args can only accept arrays. + static constexpr auto name = return_descr(props::descriptor); // Explicitly delete these: support python -> C++ conversion on these (i.e. these can be return // types but not bound arguments). We still provide them (with an explicitly delete) so that @@ -455,6 +472,8 @@ struct type_caster< using Type = Eigen::Ref; using props = EigenProps; using Scalar = typename props::Scalar; + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using MapType = Eigen::Map; using Array = array_t struct type_caster::value>> { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + protected: using Matrix = Eigen::Matrix; @@ -630,6 +652,8 @@ struct type_caster::value>> { template struct type_caster::value>> { using Scalar = typename Type::Scalar; + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using StorageIndex = remove_reference_t().outerIndexPtr())>; using Index = typename Type::Index; static constexpr bool rowMajor = Type::IsRowMajor; diff --git a/lib/pybind11/include/pybind11/eigen/tensor.h b/lib/pybind11/include/pybind11/eigen/tensor.h index 0877da8..e5c8441 100644 --- a/lib/pybind11/include/pybind11/eigen/tensor.h +++ b/lib/pybind11/include/pybind11/eigen/tensor.h @@ -7,7 +7,9 @@ #pragma once -#include "../numpy.h" +#include + +#include "common.h" #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) static_assert(__GNUC__ > 5, "Eigen Tensor support in pybind11 requires GCC > 5.0"); @@ -17,7 +19,9 @@ static_assert(__GNUC__ > 5, "Eigen Tensor support in pybind11 requires GCC > 5.0 PYBIND11_WARNING_PUSH PYBIND11_WARNING_DISABLE_MSVC(4554) PYBIND11_WARNING_DISABLE_MSVC(4127) +#if defined(__MINGW32__) PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized") +#endif #include @@ -67,7 +71,7 @@ struct eigen_tensor_helper struct helper> { - static constexpr auto value = concat(const_name(((void) Is, "?"))...); + static constexpr auto value = ::pybind11::detail::concat(const_name(((void) Is, "?"))...); }; static constexpr auto dimensions_descriptor @@ -101,7 +105,8 @@ struct eigen_tensor_helper< return get_shape() == shape; } - static constexpr auto dimensions_descriptor = concat(const_name()...); + static constexpr auto dimensions_descriptor + = ::pybind11::detail::concat(const_name()...); template static Type *alloc(Args &&...args) { @@ -119,13 +124,15 @@ struct eigen_tensor_helper< template struct get_tensor_descriptor { static constexpr auto details - = const_name(", flags.writeable", "") + = const_name(", \"flags.writeable\"", "") + const_name(Type::Layout) == static_cast(Eigen::RowMajor)>( - ", flags.c_contiguous", ", flags.f_contiguous"); + ", \"flags.c_contiguous\"", ", \"flags.f_contiguous\""); static constexpr auto value - = const_name("numpy.ndarray[") + npy_format_descriptor::name - + const_name("[") + eigen_tensor_helper>::dimensions_descriptor - + const_name("]") + const_name(details, const_name("")) + const_name("]"); + = const_name("typing.Annotated[") + + io_name("numpy.typing.ArrayLike, ", "numpy.typing.NDArray[") + + npy_format_descriptor::name + io_name("", "]") + + const_name(", \"[") + eigen_tensor_helper>::dimensions_descriptor + + const_name("]\"") + const_name(details, const_name("")) + const_name("]"); }; // When EIGEN_AVOID_STL_ARRAY is defined, Eigen::DSizes does not have the begin() member @@ -162,6 +169,8 @@ PYBIND11_WARNING_POP template struct type_caster::ValidType> { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using Helper = eigen_tensor_helper; static constexpr auto temp_name = get_tensor_descriptor::value; PYBIND11_TYPE_CASTER(Type, temp_name); @@ -357,6 +366,8 @@ struct get_storage_pointer_type struct type_caster, typename eigen_tensor_helper>::ValidType> { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using MapType = Eigen::TensorMap; using Helper = eigen_tensor_helper>; @@ -461,9 +472,6 @@ struct type_caster, parent_object = reinterpret_borrow(parent); break; - case return_value_policy::take_ownership: - delete src; - // fallthrough default: // move, take_ownership don't make any sense for a ref/map: pybind11_fail("Invalid return_value_policy for Eigen Map type, must be either " @@ -496,7 +504,10 @@ struct type_caster, std::unique_ptr value; public: - static constexpr auto name = get_tensor_descriptor::value; + // return_descr forces the use of NDArray instead of ArrayLike since refs can only reference + // arrays + static constexpr auto name + = return_descr(get_tensor_descriptor::value); explicit operator MapType *() { return value.get(); } explicit operator MapType &() { return *value; } explicit operator MapType &&() && { return std::move(*value); } diff --git a/lib/pybind11/include/pybind11/embed.h b/lib/pybind11/include/pybind11/embed.h index 749c75b..078f419 100644 --- a/lib/pybind11/include/pybind11/embed.h +++ b/lib/pybind11/include/pybind11/embed.h @@ -12,9 +12,6 @@ #include "pybind11.h" #include "eval.h" -#include -#include - #if defined(PYPY_VERSION) # error Embedding the interpreter is not supported with PyPy #endif @@ -37,24 +34,32 @@ return "Hello, World!"; }); } + + The third and subsequent macro arguments are optional, and can be used to + mark the module as supporting various Python features. + + - ``mod_gil_not_used()`` + - ``multiple_interpreters::per_interpreter_gil()`` + - ``multiple_interpreters::shared_gil()`` + - ``multiple_interpreters::not_supported()`` + + .. code-block:: cpp + + PYBIND11_EMBEDDED_MODULE(example, m, py::mod_gil_not_used()) { + m.def("foo", []() { + return "Hello, Free-threaded World!"; + }); + } + \endrst */ -#define PYBIND11_EMBEDDED_MODULE(name, variable) \ - static ::pybind11::module_::module_def PYBIND11_CONCAT(pybind11_module_def_, name); \ - static void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ &); \ - static PyObject PYBIND11_CONCAT(*pybind11_init_wrapper_, name)() { \ - auto m = ::pybind11::module_::create_extension_module( \ - PYBIND11_TOSTRING(name), nullptr, &PYBIND11_CONCAT(pybind11_module_def_, name)); \ - try { \ - PYBIND11_CONCAT(pybind11_init_, name)(m); \ - return m.ptr(); \ - } \ - PYBIND11_CATCH_INIT_EXCEPTIONS \ - } \ - PYBIND11_EMBEDDED_MODULE_IMPL(name) \ +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_CLANG("-Wgnu-zero-variadic-macro-arguments") +#define PYBIND11_EMBEDDED_MODULE(name, variable, ...) \ + PYBIND11_MODULE_PYINIT(name, ##__VA_ARGS__) \ ::pybind11::detail::embedded_module PYBIND11_CONCAT(pybind11_module_, name)( \ - PYBIND11_TOSTRING(name), PYBIND11_CONCAT(pybind11_init_impl_, name)); \ - void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ \ - & variable) // NOLINT(bugprone-macro-parentheses) + PYBIND11_TOSTRING(name), PYBIND11_CONCAT(PyInit_, name)); \ + PYBIND11_MODULE_EXEC(name, variable) +PYBIND11_WARNING_POP PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) @@ -74,73 +79,14 @@ struct embedded_module { } }; -struct wide_char_arg_deleter { - void operator()(wchar_t *ptr) const { - // API docs: https://docs.python.org/3/c-api/sys.html#c.Py_DecodeLocale - PyMem_RawFree(ptr); - } -}; - -inline wchar_t *widen_chars(const char *safe_arg) { - wchar_t *widened_arg = Py_DecodeLocale(safe_arg, nullptr); - return widened_arg; -} - inline void precheck_interpreter() { if (Py_IsInitialized() != 0) { pybind11_fail("The interpreter is already running"); } } -#if !defined(PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX) -# define PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX (0x03080000) -#endif - -#if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX -inline void initialize_interpreter_pre_pyconfig(bool init_signal_handlers, - int argc, - const char *const *argv, - bool add_program_dir_to_path) { - detail::precheck_interpreter(); - Py_InitializeEx(init_signal_handlers ? 1 : 0); -# if defined(WITH_THREAD) && PY_VERSION_HEX < 0x03070000 - PyEval_InitThreads(); -# endif - - // Before it was special-cased in python 3.8, passing an empty or null argv - // caused a segfault, so we have to reimplement the special case ourselves. - bool special_case = (argv == nullptr || argc <= 0); - - const char *const empty_argv[]{"\0"}; - const char *const *safe_argv = special_case ? empty_argv : argv; - if (special_case) { - argc = 1; - } - - auto argv_size = static_cast(argc); - // SetArgv* on python 3 takes wchar_t, so we have to convert. - std::unique_ptr widened_argv(new wchar_t *[argv_size]); - std::vector> widened_argv_entries; - widened_argv_entries.reserve(argv_size); - for (size_t ii = 0; ii < argv_size; ++ii) { - widened_argv_entries.emplace_back(detail::widen_chars(safe_argv[ii])); - if (!widened_argv_entries.back()) { - // A null here indicates a character-encoding failure or the python - // interpreter out of memory. Give up. - return; - } - widened_argv[ii] = widened_argv_entries.back().get(); - } - - auto *pysys_argv = widened_argv.get(); - - PySys_SetArgvEx(argc, pysys_argv, static_cast(add_program_dir_to_path)); -} -#endif - PYBIND11_NAMESPACE_END(detail) -#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX inline void initialize_interpreter(PyConfig *config, int argc = 0, const char *const *argv = nullptr, @@ -168,42 +114,37 @@ inline void initialize_interpreter(PyConfig *config, } PyConfig_Clear(config); } -#endif /** \rst Initialize the Python interpreter. No other pybind11 or CPython API functions can be called before this is done; with the exception of `PYBIND11_EMBEDDED_MODULE`. The optional `init_signal_handlers` parameter can be used to skip the registration of - signal handlers (see the `Python documentation`_ for details). Calling this function - again after the interpreter has already been initialized is a fatal error. + signal handlers. Calling this function again while an interpreter is running throws + ``std::runtime_error``. - If initializing the Python interpreter fails, then the program is terminated. (This - is controlled by the CPython runtime and is an exception to pybind11's normal behavior - of throwing exceptions on errors.) + The interpreter starts from a ``PyConfig``, with ``PyConfig_SetBytesArgv`` and + ``Py_InitializeFromConfig``. If the start fails, this function throws + ``std::runtime_error``. See the `Python documentation`_ for these functions. The remaining optional parameters, `argc`, `argv`, and `add_program_dir_to_path` are used to populate ``sys.argv`` and ``sys.path``. - See the |PySys_SetArgvEx documentation|_ for details. - .. _Python documentation: https://docs.python.org/3/c-api/init.html#c.Py_InitializeEx - .. |PySys_SetArgvEx documentation| replace:: ``PySys_SetArgvEx`` documentation - .. _PySys_SetArgvEx documentation: https://docs.python.org/3/c-api/init.html#c.PySys_SetArgvEx + .. _Python documentation: https://docs.python.org/3/c-api/init_config.html \endrst */ inline void initialize_interpreter(bool init_signal_handlers = true, int argc = 0, const char *const *argv = nullptr, bool add_program_dir_to_path = true) { -#if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX - detail::initialize_interpreter_pre_pyconfig( - init_signal_handlers, argc, argv, add_program_dir_to_path); -#else PyConfig config; - PyConfig_InitIsolatedConfig(&config); - config.isolated = 0; - config.use_environment = 1; + PyConfig_InitPythonConfig(&config); + // See PR #4473 for background + config.parse_argv = 0; + config.install_signal_handlers = init_signal_handlers ? 1 : 0; initialize_interpreter(&config, argc, argv, add_program_dir_to_path); -#endif + + // There is exactly one interpreter alive currently. + detail::has_seen_non_main_interpreter() = false; } /** \rst @@ -242,28 +183,32 @@ inline void initialize_interpreter(bool init_signal_handlers = true, \endrst */ inline void finalize_interpreter() { - handle builtins(PyEval_GetBuiltins()); - const char *id = PYBIND11_INTERNALS_ID; - - // Get the internals pointer (without creating it if it doesn't exist). It's possible for the - // internals to be created during Py_Finalize() (e.g. if a py::capsule calls `get_internals()` - // during destruction), so we get the pointer-pointer here and check it after Py_Finalize(). - detail::internals **internals_ptr_ptr = detail::get_internals_pp(); - // It could also be stashed in builtins, so look there too: - if (builtins.contains(id) && isinstance(builtins[id])) { - internals_ptr_ptr = capsule(builtins[id]); + // get rid of any thread-local interpreter cache that currently exists + if (detail::has_seen_non_main_interpreter()) { + detail::get_internals_pp_manager().unref(); + detail::get_local_internals_pp_manager().unref(); + + // We know there can be no other interpreter alive now + detail::has_seen_non_main_interpreter() = false; } - // Local internals contains data managed by the current interpreter, so we must clear them to - // avoid undefined behaviors when initializing another interpreter - detail::get_local_internals().registered_types_cpp.clear(); - detail::get_local_internals().registered_exception_translators.clear(); + + // Re-fetch the internals pointer-to-pointer (but not the internals itself, which might not + // exist). It's possible for the internals to be created during Py_Finalize() (e.g. if a + // py::capsule calls `get_internals()` during destruction), so we get the pointer-pointer here + // and check it after Py_Finalize(). + detail::get_internals_pp_manager().get_pp(); + detail::get_local_internals_pp_manager().get_pp(); Py_Finalize(); - if (internals_ptr_ptr) { - delete *internals_ptr_ptr; - *internals_ptr_ptr = nullptr; - } + detail::get_internals_pp_manager().destroy(); + + // Local internals contains data managed by the current interpreter, so we must clear them to + // avoid undefined behaviors when initializing another interpreter + detail::get_local_internals_pp_manager().destroy(); + + // We know there is no interpreter alive now, so we can reset the multi-flag + detail::has_seen_non_main_interpreter() = false; } /** \rst @@ -290,14 +235,12 @@ class scoped_interpreter { initialize_interpreter(init_signal_handlers, argc, argv, add_program_dir_to_path); } -#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX explicit scoped_interpreter(PyConfig *config, int argc = 0, const char *const *argv = nullptr, bool add_program_dir_to_path = true) { initialize_interpreter(config, argc, argv, add_program_dir_to_path); } -#endif scoped_interpreter(const scoped_interpreter &) = delete; scoped_interpreter(scoped_interpreter &&other) noexcept { other.is_valid = false; } diff --git a/lib/pybind11/include/pybind11/eval.h b/lib/pybind11/include/pybind11/eval.h index bd5f981..ee24fd3 100644 --- a/lib/pybind11/include/pybind11/eval.h +++ b/lib/pybind11/include/pybind11/eval.h @@ -19,7 +19,7 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) inline void ensure_builtins_in_globals(object &global) { -#if defined(PYPY_VERSION) || PY_VERSION_HEX < 0x03080000 +#if defined(PYPY_VERSION) // Running exec and eval adds `builtins` module under `__builtins__` key to // globals if not yet present. Python 3.8 made PyRun_String behave // similarly. Let's also do that for older versions, for consistency. This @@ -52,9 +52,8 @@ object eval(const str &expr, object global = globals(), object local = object()) detail::ensure_builtins_in_globals(global); - /* PyRun_String does not accept a PyObject / encoding specifier, - this seems to be the only alternative */ - std::string buffer = "# -*- coding: utf-8 -*-\n" + (std::string) expr; + // PyRun_String requires a plain C string; Python 3 always assumes UTF-8 source. + std::string buffer = (std::string) expr; int start = 0; switch (mode) { @@ -94,18 +93,18 @@ void exec(const char (&s)[N], object global = globals(), object local = object() eval(s, std::move(global), std::move(local)); } -#if defined(PYPY_VERSION) +#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) template object eval_file(str, object, object) { - pybind11_fail("eval_file not supported in PyPy3. Use eval"); + pybind11_fail("eval_file not supported in this interpreter. Use eval"); } template object eval_file(str, object) { - pybind11_fail("eval_file not supported in PyPy3. Use eval"); + pybind11_fail("eval_file not supported in this interpreter. Use eval"); } template object eval_file(str) { - pybind11_fail("eval_file not supported in PyPy3. Use eval"); + pybind11_fail("eval_file not supported in this interpreter. Use eval"); } #else template @@ -133,7 +132,12 @@ object eval_file(str fname, object global = globals(), object local = object()) int closeFile = 1; std::string fname_str = (std::string) fname; - FILE *f = _Py_fopen_obj(fname.ptr(), "r"); + FILE *f = +# if PY_VERSION_HEX >= 0x030E0000 + Py_fopen(fname.ptr(), "r"); +# else + _Py_fopen_obj(fname.ptr(), "r"); +# endif if (!f) { PyErr_Clear(); pybind11_fail("File \"" + fname_str + "\" could not be opened!"); diff --git a/lib/pybind11/include/pybind11/functional.h b/lib/pybind11/include/pybind11/functional.h index 87ec4d1..c06fabe 100644 --- a/lib/pybind11/include/pybind11/functional.h +++ b/lib/pybind11/include/pybind11/functional.h @@ -15,6 +15,41 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) +PYBIND11_NAMESPACE_BEGIN(type_caster_std_function_specializations) + +// ensure GIL is held during functor destruction +struct func_handle { + function f; + explicit func_handle(function &&f_) noexcept : f(std::move(f_)) {} + func_handle(const func_handle &f_) { operator=(f_); } + func_handle &operator=(const func_handle &f_) { + gil_scoped_acquire acq; + f = f_.f; + return *this; + } + ~func_handle() { + gil_scoped_acquire acq; + function kill_f(std::move(f)); + } +}; + +// to emulate 'move initialization capture' in C++11 +struct func_wrapper_base { + func_handle hfunc; + explicit func_wrapper_base(func_handle &&hf) noexcept : hfunc(hf) {} +}; + +template +struct func_wrapper : func_wrapper_base { + using func_wrapper_base::func_wrapper_base; + Return operator()(Args... args) const { // NOLINT(performance-unnecessary-value-param) + gil_scoped_acquire acq; + // casts the returned object as a rvalue to the return type + return hfunc.f(std::forward(args)...).template cast(); + } +}; + +PYBIND11_NAMESPACE_END(type_caster_std_function_specializations) template struct type_caster> { @@ -50,23 +85,22 @@ struct type_caster> { auto *cfunc_self = PyCFunction_GET_SELF(cfunc.ptr()); if (cfunc_self == nullptr) { PyErr_Clear(); - } else if (isinstance(cfunc_self)) { - auto c = reinterpret_borrow(cfunc_self); - - function_record *rec = nullptr; - // Check that we can safely reinterpret the capsule into a function_record - if (detail::is_function_record_capsule(c)) { - rec = c.get_pointer(); - } - + } else { + function_record *rec = function_record_ptr_from_PyObject(cfunc_self); while (rec != nullptr) { if (rec->is_stateless && same_type(typeid(function_type), *reinterpret_cast(rec->data[1]))) { struct capture { function_type f; + + static capture *from_data(void **data) { + return PYBIND11_STD_LAUNDER(reinterpret_cast(data)); + } }; - value = ((capture *) &rec->data)->f; + PYBIND11_ENSURE_PRECONDITION_FOR_FUNCTIONAL_H_PERFORMANCE_OPTIMIZATIONS( + std::is_standard_layout::value); + value = capture::from_data(rec->data)->f; return true; } rec = rec->next; @@ -77,40 +111,8 @@ struct type_caster> { // See PR #1413 for full details } - // ensure GIL is held during functor destruction - struct func_handle { - function f; -#if !(defined(_MSC_VER) && _MSC_VER == 1916 && defined(PYBIND11_CPP17)) - // This triggers a syntax error under very special conditions (very weird indeed). - explicit -#endif - func_handle(function &&f_) noexcept - : f(std::move(f_)) { - } - func_handle(const func_handle &f_) { operator=(f_); } - func_handle &operator=(const func_handle &f_) { - gil_scoped_acquire acq; - f = f_.f; - return *this; - } - ~func_handle() { - gil_scoped_acquire acq; - function kill_f(std::move(f)); - } - }; - - // to emulate 'move initialization capture' in C++11 - struct func_wrapper { - func_handle hfunc; - explicit func_wrapper(func_handle &&hf) noexcept : hfunc(std::move(hf)) {} - Return operator()(Args... args) const { - gil_scoped_acquire acq; - // casts the returned object as a rvalue to the return type - return hfunc.f(std::forward(args)...).template cast(); - } - }; - - value = func_wrapper(func_handle(std::move(func))); + value = type_caster_std_function_specializations::func_wrapper( + type_caster_std_function_specializations::func_handle(std::move(func))); return true; } @@ -127,10 +129,11 @@ struct type_caster> { return cpp_function(std::forward(f_), policy).release(); } - PYBIND11_TYPE_CASTER(type, - const_name("Callable[[") + concat(make_caster::name...) - + const_name("], ") + make_caster::name - + const_name("]")); + PYBIND11_TYPE_CASTER( + type, + const_name("collections.abc.Callable[[") + + ::pybind11::detail::concat(::pybind11::detail::inv_descr(make_caster::name)...) + + const_name("], ") + make_caster::name + const_name("]")); }; PYBIND11_NAMESPACE_END(detail) diff --git a/lib/pybind11/include/pybind11/gil.h b/lib/pybind11/include/pybind11/gil.h index cb0028d..9e799b3 100644 --- a/lib/pybind11/include/pybind11/gil.h +++ b/lib/pybind11/include/pybind11/gil.h @@ -9,24 +9,38 @@ #pragma once -#include "detail/common.h" +#if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) -#if defined(WITH_THREAD) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +# include "detail/common.h" +# include "gil_simple.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +using gil_scoped_acquire = gil_scoped_acquire_simple; +using gil_scoped_release = gil_scoped_release_simple; + +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) + +#else + +# include "detail/common.h" # include "detail/internals.h" -#endif + +# include PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_GCC("-Wredundant-decls") + // forward declarations PyThreadState *get_thread_state_unchecked(); -PYBIND11_NAMESPACE_END(detail) - -#if defined(WITH_THREAD) +PYBIND11_WARNING_POP -# if !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +PYBIND11_NAMESPACE_END(detail) /* The functions below essentially reproduce the PyGILState_* API using a RAII * pattern, but there are a few important differences: @@ -54,7 +68,7 @@ class gil_scoped_acquire { public: PYBIND11_NOINLINE gil_scoped_acquire() { auto &internals = detail::get_internals(); - tstate = (PyThreadState *) PYBIND11_TLS_GET_VALUE(internals.tstate); + tstate = internals.tstate.get(); if (!tstate) { /* Check if the GIL was acquired using the PyGILState_* API instead (e.g. if @@ -67,13 +81,13 @@ class gil_scoped_acquire { if (!tstate) { tstate = PyThreadState_New(internals.istate); -# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) +# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) if (!tstate) { pybind11_fail("scoped_acquire: could not create thread state!"); } -# endif +# endif tstate->gilstate_counter = 0; - PYBIND11_TLS_REPLACE_VALUE(internals.tstate, tstate); + internals.tstate = tstate; } else { release = detail::get_thread_state_unchecked() != tstate; } @@ -92,31 +106,35 @@ class gil_scoped_acquire { PYBIND11_NOINLINE void dec_ref() { --tstate->gilstate_counter; -# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) +# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) if (detail::get_thread_state_unchecked() != tstate) { pybind11_fail("scoped_acquire::dec_ref(): thread state must be current!"); } if (tstate->gilstate_counter < 0) { pybind11_fail("scoped_acquire::dec_ref(): reference count underflow!"); } -# endif +# endif if (tstate->gilstate_counter == 0) { -# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) +# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) if (!release) { pybind11_fail("scoped_acquire::dec_ref(): internal error!"); } -# endif +# endif + // Make sure that PyThreadState_Clear is not recursively called by finalizers. + // See issue #5827 + ++tstate->gilstate_counter; PyThreadState_Clear(tstate); + --tstate->gilstate_counter; if (active) { PyThreadState_DeleteCurrent(); } - PYBIND11_TLS_DELETE_VALUE(detail::get_internals().tstate); + detail::get_internals().tstate.reset(); release = false; } } /// This method will disable the PyThreadState_DeleteCurrent call and the - /// GIL won't be acquired. This method should be used if the interpreter + /// GIL won't be released. This method should be used if the interpreter /// could be shutting down when this is called, as thread deletion is not /// allowed during shutdown. Check _Py_IsFinalizing() on Python 3.7+, and /// protect subsequent code. @@ -137,7 +155,9 @@ class gil_scoped_acquire { class gil_scoped_release { public: + // PRECONDITION: The GIL must be held when this constructor is called. explicit gil_scoped_release(bool disassoc = false) : disassoc(disassoc) { + assert(PyGILState_Check()); // `get_internals()` must be called here unconditionally in order to initialize // `internals.tstate` for subsequent `gil_scoped_acquire` calls. Otherwise, an // initialization race could occur as multiple threads try `gil_scoped_acquire`. @@ -145,15 +165,12 @@ class gil_scoped_release { // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) tstate = PyEval_SaveThread(); if (disassoc) { - // Python >= 3.7 can remove this, it's an int before 3.7 - // NOLINTNEXTLINE(readability-qualified-auto) - auto key = internals.tstate; - PYBIND11_TLS_DELETE_VALUE(key); + internals.tstate.reset(); } } - gil_scoped_release(const gil_scoped_acquire &) = delete; - gil_scoped_release &operator=(const gil_scoped_acquire &) = delete; + gil_scoped_release(const gil_scoped_release &) = delete; + gil_scoped_release &operator=(const gil_scoped_release &) = delete; /// This method will disable the PyThreadState_DeleteCurrent call and the /// GIL won't be acquired. This method should be used if the interpreter @@ -171,10 +188,7 @@ class gil_scoped_release { PyEval_RestoreThread(tstate); } if (disassoc) { - // Python >= 3.7 can remove this, it's an int before 3.7 - // NOLINTNEXTLINE(readability-qualified-auto) - auto key = detail::get_internals().tstate; - PYBIND11_TLS_REPLACE_VALUE(key, tstate); + detail::get_internals().tstate = tstate; } } @@ -184,56 +198,6 @@ class gil_scoped_release { bool active = true; }; -# else // PYBIND11_SIMPLE_GIL_MANAGEMENT - -class gil_scoped_acquire { - PyGILState_STATE state; - -public: - gil_scoped_acquire() : state{PyGILState_Ensure()} {} - gil_scoped_acquire(const gil_scoped_acquire &) = delete; - gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete; - ~gil_scoped_acquire() { PyGILState_Release(state); } - void disarm() {} -}; - -class gil_scoped_release { - PyThreadState *state; - -public: - gil_scoped_release() : state{PyEval_SaveThread()} {} - gil_scoped_release(const gil_scoped_release &) = delete; - gil_scoped_release &operator=(const gil_scoped_acquire &) = delete; - ~gil_scoped_release() { PyEval_RestoreThread(state); } - void disarm() {} -}; - -# endif // PYBIND11_SIMPLE_GIL_MANAGEMENT - -#else // WITH_THREAD - -class gil_scoped_acquire { -public: - gil_scoped_acquire() { - // Trick to suppress `unused variable` error messages (at call sites). - (void) (this != (this + 1)); - } - gil_scoped_acquire(const gil_scoped_acquire &) = delete; - gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete; - void disarm() {} -}; - -class gil_scoped_release { -public: - gil_scoped_release() { - // Trick to suppress `unused variable` error messages (at call sites). - (void) (this != (this + 1)); - } - gil_scoped_release(const gil_scoped_release &) = delete; - gil_scoped_release &operator=(const gil_scoped_acquire &) = delete; - void disarm() {} -}; - -#endif // WITH_THREAD - PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) + +#endif // !PYBIND11_SIMPLE_GIL_MANAGEMENT diff --git a/lib/pybind11/include/pybind11/gil_safe_call_once.h b/lib/pybind11/include/pybind11/gil_safe_call_once.h new file mode 100644 index 0000000..af26272 --- /dev/null +++ b/lib/pybind11/include/pybind11/gil_safe_call_once.h @@ -0,0 +1,283 @@ +// Copyright (c) 2023 The pybind Community. + +#pragma once + +#include "detail/common.h" +#include "detail/internals.h" +#include "gil.h" + +#include +#include + +#if defined(Py_GIL_DISABLED) || defined(PYBIND11_HAS_SUBINTERPRETER_SUPPORT) +# include +#endif +#ifdef PYBIND11_HAS_SUBINTERPRETER_SUPPORT +# include +# include +# include +#endif + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_NAMESPACE_BEGIN(detail) +#if defined(Py_GIL_DISABLED) || defined(PYBIND11_HAS_SUBINTERPRETER_SUPPORT) +using atomic_bool = std::atomic_bool; +#else +using atomic_bool = bool; +#endif +PYBIND11_NAMESPACE_END(detail) + +// Use the `gil_safe_call_once_and_store` class below instead of the naive +// +// static auto imported_obj = py::module_::import("module_name"); // BAD, DO NOT USE! +// +// which has two serious issues: +// +// 1. Py_DECREF() calls potentially after the Python interpreter was finalized already, and +// 2. deadlocks in multi-threaded processes (because of missing lock ordering). +// +// The following alternative avoids both problems: +// +// PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store storage; +// auto &imported_obj = storage // Do NOT make this `static`! +// .call_once_and_store_result([]() { +// return py::module_::import("module_name"); +// }) +// .get_stored(); +// +// The parameter of `call_once_and_store_result()` must be callable. It can make +// CPython API calls, and in particular, it can temporarily release the GIL. +// +// `T` can be any C++ type, it does not have to involve CPython API types. +// +// The behavior with regard to signals, e.g. `SIGINT` (`KeyboardInterrupt`), +// is not ideal. If the main thread is the one to actually run the `Callable`, +// then a `KeyboardInterrupt` will interrupt it if it is running normal Python +// code. The situation is different if a non-main thread runs the +// `Callable`, and then the main thread starts waiting for it to complete: +// a `KeyboardInterrupt` will not interrupt the non-main thread, but it will +// get processed only when it is the main thread's turn again and it is running +// normal Python code. However, this will be unnoticeable for quick call-once +// functions, which is usually the case. +// +// For in-depth background, see docs/advanced/deadlock.md +#ifndef PYBIND11_HAS_SUBINTERPRETER_SUPPORT +// Subinterpreter support is disabled. +// In this case, we can store the result globally, because there is only a single interpreter. +// +// The life span of the stored result is the entire process lifetime. It is leaked on process +// termination to avoid destructor calls after the Python interpreter was finalized. +template +class gil_safe_call_once_and_store { +public: + // PRECONDITION: The GIL must be held when `call_once_and_store_result()` is called. + // + // NOTE: The second parameter (finalize callback) is intentionally unused when subinterpreter + // support is disabled. In that case, storage is process-global and intentionally leaked to + // avoid calling destructors after the Python interpreter has been finalized. + template + gil_safe_call_once_and_store &call_once_and_store_result(Callable &&fn, + void (*)(T &) /*unused*/ = nullptr) { + if (!is_initialized_) { // This read is guarded by the GIL. + // Multiple threads may enter here, because the GIL is released in the next line and + // CPython API calls in the `fn()` call below may release and reacquire the GIL. + gil_scoped_release gil_rel; // Needed to establish lock ordering. + std::call_once(once_flag_, [&] { + // Only one thread will ever enter here. + gil_scoped_acquire gil_acq; + ::new (storage_) T(fn()); // fn may release, but will reacquire, the GIL. + is_initialized_ = true; // This write is guarded by the GIL. + }); + // All threads will observe `is_initialized_` as true here. + } + // Intentionally not returning `T &` to ensure the calling code is self-documenting. + return *this; + } + + // This must only be called after `call_once_and_store_result()` was called. + T &get_stored() { + assert(is_initialized_); + PYBIND11_WARNING_PUSH +# if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 5 + // Needed for gcc 4.8.5 + PYBIND11_WARNING_DISABLE_GCC("-Wstrict-aliasing") +# endif + return *reinterpret_cast(storage_); + PYBIND11_WARNING_POP + } + + constexpr gil_safe_call_once_and_store() = default; + // The instance is a global static, so its destructor runs when the process + // is terminating. Therefore, do nothing here because the Python interpreter + // may have been finalized already. + PYBIND11_DTOR_CONSTEXPR ~gil_safe_call_once_and_store() = default; + + // Disable copy and move operations. + gil_safe_call_once_and_store(const gil_safe_call_once_and_store &) = delete; + gil_safe_call_once_and_store(gil_safe_call_once_and_store &&) = delete; + gil_safe_call_once_and_store &operator=(const gil_safe_call_once_and_store &) = delete; + gil_safe_call_once_and_store &operator=(gil_safe_call_once_and_store &&) = delete; + +private: + // The global static storage (per-process) when subinterpreter support is disabled. + alignas(T) char storage_[sizeof(T)] = {}; + std::once_flag once_flag_; + + // The `is_initialized_`-`storage_` pair is very similar to `std::optional`, + // but the latter does not have the triviality properties of former, + // therefore `std::optional` is not a viable alternative here. + detail::atomic_bool is_initialized_{false}; +}; +#else +// Subinterpreter support is enabled. +// In this case, we should store the result per-interpreter instead of globally, because each +// subinterpreter has its own separate state. The cached result may not shareable across +// interpreters (e.g., imported modules and their members). + +PYBIND11_NAMESPACE_BEGIN(detail) + +template +struct call_once_storage { + alignas(T) char storage[sizeof(T)] = {}; + std::once_flag once_flag; + void (*finalize)(T &) = nullptr; + std::atomic_bool is_initialized{false}; + + call_once_storage() = default; + ~call_once_storage() { + if (is_initialized) { + if (finalize != nullptr) { + finalize(*reinterpret_cast(storage)); + } else { + reinterpret_cast(storage)->~T(); + } + } + } + call_once_storage(const call_once_storage &) = delete; + call_once_storage(call_once_storage &&) = delete; + call_once_storage &operator=(const call_once_storage &) = delete; + call_once_storage &operator=(call_once_storage &&) = delete; +}; + +PYBIND11_NAMESPACE_END(detail) + +// Prefix for storage keys in the interpreter state dict. +# define PYBIND11_CALL_ONCE_STORAGE_KEY_PREFIX PYBIND11_INTERNALS_ID "_call_once_storage__" + +// The life span of the stored result is the entire interpreter lifetime. An additional +// `finalize_fn` can be provided to clean up the stored result when the interpreter is destroyed. +template +class gil_safe_call_once_and_store { +public: + // PRECONDITION: The GIL must be held when `call_once_and_store_result()` is called. + template + gil_safe_call_once_and_store &call_once_and_store_result(Callable &&fn, + void (*finalize_fn)(T &) = nullptr) { + if (!is_last_storage_valid()) { + // Multiple threads may enter here, because the GIL is released in the next line and + // CPython API calls in the `fn()` call below may release and reacquire the GIL. + gil_scoped_release gil_rel; // Needed to establish lock ordering. + // There can be multiple threads going through here. + storage_type *value = nullptr; + { + gil_scoped_acquire gil_acq; // Restore lock ordering. + // This function is thread-safe under free-threading. + value = get_or_create_storage_in_state_dict(); + } + assert(value != nullptr); + std::call_once(value->once_flag, [&] { + // Only one thread will ever enter here. + gil_scoped_acquire gil_acq; + // fn may release, but will reacquire, the GIL. + ::new (value->storage) T(fn()); + value->finalize = finalize_fn; + value->is_initialized = true; + // Publish the cached pointer before setting the validity flag, so that any reader + // which observes the flag as true is guaranteed to also observe this pointer. + last_storage_ptr_ = reinterpret_cast(value->storage); + is_initialized_by_at_least_one_interpreter_ = true; + }); + // All threads will observe `is_initialized_by_at_least_one_interpreter_` as true here. + } + // Intentionally not returning `T &` to ensure the calling code is self-documenting. + return *this; + } + + // This must only be called after `call_once_and_store_result()` was called. + T &get_stored() { + T *result = nullptr; + // Check the validity flag *before* loading the cached pointer: the writer publishes the + // pointer before setting the flag, so observing the flag as true guarantees the load below + // sees the published pointer. + if (is_last_storage_valid()) { + result = last_storage_ptr_; + } else { + gil_scoped_acquire gil_acq; + auto *value = get_or_create_storage_in_state_dict(); + result = reinterpret_cast(value->storage); + last_storage_ptr_ = result; + } + assert(result != nullptr); + return *result; + } + + constexpr gil_safe_call_once_and_store() = default; + // The instance is a global static, so its destructor runs when the process + // is terminating. Therefore, do nothing here because the Python interpreter + // may have been finalized already. + PYBIND11_DTOR_CONSTEXPR ~gil_safe_call_once_and_store() = default; + + // Disable copy and move operations because the memory address is used as key. + gil_safe_call_once_and_store(const gil_safe_call_once_and_store &) = delete; + gil_safe_call_once_and_store(gil_safe_call_once_and_store &&) = delete; + gil_safe_call_once_and_store &operator=(const gil_safe_call_once_and_store &) = delete; + gil_safe_call_once_and_store &operator=(gil_safe_call_once_and_store &&) = delete; + +private: + using storage_type = detail::call_once_storage; + + // Indicator of fast path for single-interpreter case. + bool is_last_storage_valid() const { + return is_initialized_by_at_least_one_interpreter_ + && !detail::has_seen_non_main_interpreter(); + } + + // Get the unique key for this storage instance in the interpreter's state dict. + // The return type should not be `py::str` because PyObject is interpreter-dependent. + std::string get_storage_key() const { + // The instance is expected to be global static, so using its address as unique identifier. + // The typical usage is like: + // + // PYBIND11_CONSTINIT static gil_safe_call_once_and_store storage; + // + return PYBIND11_CALL_ONCE_STORAGE_KEY_PREFIX + + std::to_string(reinterpret_cast(this)); + } + + // Get or create per-storage capsule in the current interpreter's state dict. + // The storage is interpreter-dependent and will not be shared across interpreters. + storage_type *get_or_create_storage_in_state_dict() { + return detail::atomic_get_or_create_in_state_dict(get_storage_key().c_str()) + .first; + } + + // No storage needed when subinterpreter support is enabled. + // The actual storage is stored in the per-interpreter state dict via + // `get_or_create_storage_in_state_dict()`. + + // Fast local cache to avoid repeated lookups when there are no multiple interpreters. + // This is only valid if there is a single interpreter. Otherwise, it is not used. + // It is atomic because under free-threading (`Py_GIL_DISABLED`) the GIL provides no mutual + // exclusion, so this pointer may be read and written concurrently by multiple threads. + // WARNING: We cannot use thread local cache similar to `internals_pp_manager::internals_p_tls` + // because the thread local storage cannot be explicitly invalidated when interpreters + // are destroyed (unlike `internals_pp_manager` which has explicit hooks for that). + std::atomic last_storage_ptr_{nullptr}; + // This flag is true if the value has been initialized by any interpreter (may not be the + // current one). + detail::atomic_bool is_initialized_by_at_least_one_interpreter_{false}; +}; +#endif + +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/gil_simple.h b/lib/pybind11/include/pybind11/gil_simple.h new file mode 100644 index 0000000..9ff428a --- /dev/null +++ b/lib/pybind11/include/pybind11/gil_simple.h @@ -0,0 +1,37 @@ +// Copyright (c) 2016-2025 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#pragma once + +#include "detail/common.h" + +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +class gil_scoped_acquire_simple { + PyGILState_STATE state; + +public: + gil_scoped_acquire_simple() : state{PyGILState_Ensure()} {} + gil_scoped_acquire_simple(const gil_scoped_acquire_simple &) = delete; + gil_scoped_acquire_simple &operator=(const gil_scoped_acquire_simple &) = delete; + ~gil_scoped_acquire_simple() { PyGILState_Release(state); } +}; + +class gil_scoped_release_simple { + PyThreadState *state; + +public: + // PRECONDITION: The GIL must be held when this constructor is called. + gil_scoped_release_simple() { + assert(PyGILState_Check()); + state = PyEval_SaveThread(); + } + gil_scoped_release_simple(const gil_scoped_release_simple &) = delete; + gil_scoped_release_simple &operator=(const gil_scoped_release_simple &) = delete; + ~gil_scoped_release_simple() { PyEval_RestoreThread(state); } +}; + +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/iostream.h b/lib/pybind11/include/pybind11/iostream.h index 1878089..df7fa3c 100644 --- a/lib/pybind11/include/pybind11/iostream.h +++ b/lib/pybind11/include/pybind11/iostream.h @@ -117,13 +117,36 @@ class pythonbuf : public std::streambuf { int sync() override { return _sync(); } public: + // Minimum buffer size must accommodate the largest incomplete UTF-8 sequence + // (3 bytes) plus one position reserved for overflow(), i.e. 4 bytes total. + static constexpr size_t minimum_buffer_size = 4; + explicit pythonbuf(const object &pyostream, size_t buffer_size = 1024) - : buf_size(buffer_size), d_buffer(new char[buf_size]), pywrite(pyostream.attr("write")), + : buf_size(buffer_size < minimum_buffer_size // ternary avoids C++14 std::max ODR-use of + // static constexpr + ? minimum_buffer_size + : buffer_size), + d_buffer(new char[buf_size]), pywrite(pyostream.attr("write")), pyflush(pyostream.attr("flush")) { setp(d_buffer.get(), d_buffer.get() + buf_size - 1); } - pythonbuf(pythonbuf &&) = default; + pythonbuf(pythonbuf &&other) noexcept + : buf_size(other.buf_size), d_buffer(std::move(other.d_buffer)), + pywrite(std::move(other.pywrite)), pyflush(std::move(other.pyflush)) { + const auto pending = (other.pbase() != nullptr && other.pptr() != nullptr) + ? static_cast(other.pptr() - other.pbase()) + : 0; + if (d_buffer != nullptr) { + // Rebuild the put area from the transferred storage. + setp(d_buffer.get(), d_buffer.get() + buf_size - 1); + pbump(pending); + } else { + setp(nullptr, nullptr); + } + // Prevent the moved-from destructor from flushing through moved-out handles. + other.setp(nullptr, nullptr); + } /// Sync before destroy ~pythonbuf() override { _sync(); } @@ -161,6 +184,7 @@ class scoped_ostream_redirect { std::streambuf *old; std::ostream &costream; detail::pythonbuf buffer; + bool active = true; public: explicit scoped_ostream_redirect(std::ostream &costream = std::cout, @@ -170,10 +194,22 @@ class scoped_ostream_redirect { old = costream.rdbuf(&buffer); } - ~scoped_ostream_redirect() { costream.rdbuf(old); } + ~scoped_ostream_redirect() { + if (active) { + costream.rdbuf(old); + } + } scoped_ostream_redirect(const scoped_ostream_redirect &) = delete; - scoped_ostream_redirect(scoped_ostream_redirect &&other) = default; + // NOLINTNEXTLINE(performance-noexcept-move-constructor) + scoped_ostream_redirect(scoped_ostream_redirect &&other) + : old(other.old), costream(other.costream), buffer(std::move(other.buffer)), + active(other.active) { + if (active) { + costream.rdbuf(&buffer); // Re-point stream to our buffer + other.active = false; + } + } scoped_ostream_redirect &operator=(const scoped_ostream_redirect &) = delete; scoped_ostream_redirect &operator=(scoped_ostream_redirect &&) = delete; }; diff --git a/lib/pybind11/include/pybind11/native_enum.h b/lib/pybind11/include/pybind11/native_enum.h new file mode 100644 index 0000000..af166d0 --- /dev/null +++ b/lib/pybind11/include/pybind11/native_enum.h @@ -0,0 +1,76 @@ +// Copyright (c) 2022-2025 The pybind Community. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#pragma once + +#include "detail/common.h" +#include "detail/native_enum_data.h" +#include "detail/type_caster_base.h" +#include "cast.h" + +#include +#include +#include +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +/// Conversions between Python's native (stdlib) enum types and C++ enums. +template +class native_enum : public detail::native_enum_data { +public: + using Underlying = typename std::underlying_type::type; + + native_enum(handle parent_scope, + const char *name, + const char *native_type_name, + const char *class_doc = "") + : detail::native_enum_data( + parent_scope, name, native_type_name, class_doc, make_record()) { + if (detail::get_local_type_info(typeid(EnumType)) != nullptr + || detail::get_global_type_info(typeid(EnumType)) != nullptr) { + pybind11_fail( + "pybind11::native_enum<...>(\"" + enum_name_encoded + + "\") is already registered as a `pybind11::enum_` or `pybind11::class_`!"); + } + if (detail::global_internals_native_enum_type_map_contains(enum_type_index)) { + pybind11_fail("pybind11::native_enum<...>(\"" + enum_name_encoded + + "\") is already registered!"); + } + arm_finalize_check(); + } + + /// Export enumeration entries into the parent scope + native_enum &export_values() { + assert(!export_values_flag); // Catch redundant calls. + export_values_flag = true; + return *this; + } + + /// Add an enumeration entry + native_enum &value(char const *name, EnumType value, const char *doc = nullptr) { + // Disarm for the case that the native_enum_data dtor runs during exception unwinding. + disarm_finalize_check("value after finalize"); + members.append(make_tuple(name, static_cast(value))); + if (doc) { + member_docs.append(make_tuple(name, doc)); + } + arm_finalize_check(); // There was no exception. + return *this; + } + + native_enum(const native_enum &) = delete; + native_enum &operator=(const native_enum &) = delete; + +private: + static detail::native_enum_record make_record() { + detail::native_enum_record ret; + ret.cpptype = &typeid(EnumType); + ret.size_bytes = sizeof(EnumType); + ret.is_signed = std::is_signed::value; + return ret; + } +}; + +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/numpy.h b/lib/pybind11/include/pybind11/numpy.h index 8f072af..10c0c94 100644 --- a/lib/pybind11/include/pybind11/numpy.h +++ b/lib/pybind11/include/pybind11/numpy.h @@ -10,7 +10,10 @@ #pragma once #include "pybind11.h" +#include "detail/common.h" #include "complex.h" +#include "gil_safe_call_once.h" +#include "pytypes.h" #include #include @@ -26,10 +29,19 @@ #include #include +#ifdef PYBIND11_HAS_SPAN +# include +#endif + +#if defined(PYBIND11_NUMPY_1_ONLY) +# error "PYBIND11_NUMPY_1_ONLY is no longer supported (see PR #5595)." +#endif + /* This will be true on all flat address space platforms and allows us to reduce the whole npy_intp / ssize_t / Py_intptr_t business down to just ssize_t for all size and dimension types (e.g. shape, strides, indexing), instead of inflicting this - upon the library user. */ + upon the library user. + Note that NumPy 2 now uses ssize_t for `npy_intp` to simplify this. */ static_assert(sizeof(::pybind11::ssize_t) == sizeof(Py_intptr_t), "ssize_t != Py_intptr_t"); static_assert(std::is_signed::value, "Py_intptr_t must be signed"); // We now can reinterpret_cast between py::ssize_t and Py_intptr_t (MSVC + PyPy cares) @@ -38,10 +50,19 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_WARNING_DISABLE_MSVC(4127) +class dtype; // Forward declaration class array; // Forward declaration +template +struct numpy_scalar; // Forward declaration + PYBIND11_NAMESPACE_BEGIN(detail) +template <> +struct handle_type_name { + static constexpr auto name = const_name("numpy.dtype"); +}; + template <> struct handle_type_name { static constexpr auto name = const_name("numpy.ndarray"); @@ -50,7 +71,8 @@ struct handle_type_name { template struct npy_format_descriptor; -struct PyArrayDescr_Proxy { +/* NumPy 1 proxy (always includes legacy fields) */ +struct PyArrayDescr1_Proxy { PyObject_HEAD PyObject *typeobj; char kind; @@ -65,6 +87,38 @@ struct PyArrayDescr_Proxy { PyObject *names; }; +struct PyArrayDescr_Proxy { + PyObject_HEAD + PyObject *typeobj; + char kind; + char type; + char byteorder; + char _former_flags; + int type_num; + /* Additional fields are NumPy version specific. */ +}; + +/* NumPy 2 proxy, including legacy fields */ +struct PyArrayDescr2_Proxy { + PyObject_HEAD + PyObject *typeobj; + char kind; + char type; + char byteorder; + char _former_flags; + int type_num; + std::uint64_t flags; + ssize_t elsize; + ssize_t alignment; + PyObject *metadata; + Py_hash_t hash; + void *reserved_null[2]; + /* The following fields only exist if 0 <= type_num < 2056 */ + char *subarray; + PyObject *fields; + PyObject *names; +}; + struct PyArray_Proxy { PyObject_HEAD char *data; @@ -120,6 +174,19 @@ inline numpy_internals &get_numpy_internals() { return *ptr; } +PYBIND11_NOINLINE module_ import_numpy_core_submodule(const char *submodule_name) { + module_ numpy = module_::import("numpy"); + str version_string = numpy.attr("__version__"); + module_ numpy_lib = module_::import("numpy.lib"); + object numpy_version = numpy_lib.attr("NumpyVersion")(version_string); + int major_version = numpy_version.attr("major").cast(); + + /* `numpy.core` was renamed to `numpy._core` in NumPy 2.0 as it officially + became a private module. */ + std::string numpy_core_path = major_version >= 2 ? "numpy._core" : "numpy.core"; + return module_::import((numpy_core_path + "." + submodule_name).c_str()); +} + template struct same_size { template @@ -138,6 +205,7 @@ constexpr int platform_lookup(int I, Ints... Is) { } struct npy_api { + // If you change this code, please review `normalized_dtype_num` below. enum constants { NPY_ARRAY_C_CONTIGUOUS_ = 0x0001, NPY_ARRAY_F_CONTIGUOUS_ = 0x0002, @@ -184,16 +252,33 @@ struct npy_api { NPY_UINT64_ = platform_lookup( NPY_ULONG_, NPY_ULONGLONG_, NPY_UINT_), + NPY_FLOAT32_ = platform_lookup( + NPY_DOUBLE_, NPY_FLOAT_, NPY_LONGDOUBLE_), + NPY_FLOAT64_ = platform_lookup( + NPY_DOUBLE_, NPY_FLOAT_, NPY_LONGDOUBLE_), + NPY_COMPLEX64_ + = platform_lookup, + std::complex, + std::complex, + std::complex>(NPY_DOUBLE_, NPY_FLOAT_, NPY_LONGDOUBLE_), + NPY_COMPLEX128_ + = platform_lookup, + std::complex, + std::complex, + std::complex>(NPY_DOUBLE_, NPY_FLOAT_, NPY_LONGDOUBLE_), + NPY_CHAR_ = std::is_signed::value ? NPY_BYTE_ : NPY_UBYTE_, }; + unsigned int PyArray_RUNTIME_VERSION_; + struct PyArray_Dims { Py_intptr_t *ptr; int len; }; static npy_api &get() { - static npy_api api = lookup(); - return api; + PYBIND11_CONSTINIT static gil_safe_call_once_and_store storage; + return storage.call_once_and_store_result(lookup).get_stored(); } bool PyArray_Check_(PyObject *obj) const { @@ -205,6 +290,7 @@ struct npy_api { unsigned int (*PyArray_GetNDArrayCFeatureVersion_)(); PyObject *(*PyArray_DescrFromType_)(int); + PyObject *(*PyArray_TypeObjectFromType_)(int); PyObject *(*PyArray_NewFromDescr_)(PyTypeObject *, PyObject *, int, @@ -221,17 +307,11 @@ struct npy_api { PyTypeObject *PyVoidArrType_Type_; PyTypeObject *PyArrayDescr_Type_; PyObject *(*PyArray_DescrFromScalar_)(PyObject *); + PyObject *(*PyArray_Scalar_)(void *, PyObject *, PyObject *); + void (*PyArray_ScalarAsCtype_)(PyObject *, void *); PyObject *(*PyArray_FromAny_)(PyObject *, PyObject *, int, int, int, PyObject *); int (*PyArray_DescrConverter_)(PyObject *, PyObject **); bool (*PyArray_EquivTypes_)(PyObject *, PyObject *); - int (*PyArray_GetArrayParamsFromObject_)(PyObject *, - PyObject *, - unsigned char, - PyObject **, - int *, - Py_intptr_t *, - PyObject **, - PyObject *); PyObject *(*PyArray_Squeeze_)(PyObject *); // Unused. Not removed because that affects ABI of the class. int (*PyArray_SetBaseObject_)(PyObject *, PyObject *); @@ -246,10 +326,14 @@ struct npy_api { API_PyArrayDescr_Type = 3, API_PyVoidArrType_Type = 39, API_PyArray_DescrFromType = 45, + API_PyArray_TypeObjectFromType = 46, API_PyArray_DescrFromScalar = 57, + API_PyArray_Scalar = 60, + API_PyArray_ScalarAsCtype = 62, API_PyArray_FromAny = 69, API_PyArray_Resize = 80, - API_PyArray_CopyInto = 82, + // CopyInto was slot 82 and 50 was effectively an alias. NumPy 2 removed 82. + API_PyArray_CopyInto = 50, API_PyArray_NewCopy = 85, API_PyArray_NewFromDescr = 94, API_PyArray_DescrNewFromType = 96, @@ -258,25 +342,32 @@ struct npy_api { API_PyArray_View = 137, API_PyArray_DescrConverter = 174, API_PyArray_EquivTypes = 182, - API_PyArray_GetArrayParamsFromObject = 278, API_PyArray_SetBaseObject = 282 }; static npy_api lookup() { - module_ m = module_::import("numpy.core.multiarray"); + module_ m = detail::import_numpy_core_submodule("multiarray"); auto c = m.attr("_ARRAY_API"); void **api_ptr = (void **) PyCapsule_GetPointer(c.ptr(), nullptr); + if (api_ptr == nullptr) { + raise_from(PyExc_SystemError, "FAILURE obtaining numpy _ARRAY_API pointer."); + throw error_already_set(); + } npy_api api; #define DECL_NPY_API(Func) api.Func##_ = (decltype(api.Func##_)) api_ptr[API_##Func]; DECL_NPY_API(PyArray_GetNDArrayCFeatureVersion); - if (api.PyArray_GetNDArrayCFeatureVersion_() < 0x7) { + api.PyArray_RUNTIME_VERSION_ = api.PyArray_GetNDArrayCFeatureVersion_(); + if (api.PyArray_RUNTIME_VERSION_ < 0x7) { pybind11_fail("pybind11 numpy support requires numpy >= 1.7.0"); } DECL_NPY_API(PyArray_Type); DECL_NPY_API(PyVoidArrType_Type); DECL_NPY_API(PyArrayDescr_Type); DECL_NPY_API(PyArray_DescrFromType); + DECL_NPY_API(PyArray_TypeObjectFromType); DECL_NPY_API(PyArray_DescrFromScalar); + DECL_NPY_API(PyArray_Scalar); + DECL_NPY_API(PyArray_ScalarAsCtype); DECL_NPY_API(PyArray_FromAny); DECL_NPY_API(PyArray_Resize); DECL_NPY_API(PyArray_CopyInto); @@ -288,7 +379,6 @@ struct npy_api { DECL_NPY_API(PyArray_View); DECL_NPY_API(PyArray_DescrConverter); DECL_NPY_API(PyArray_EquivTypes); - DECL_NPY_API(PyArray_GetArrayParamsFromObject); DECL_NPY_API(PyArray_SetBaseObject); #undef DECL_NPY_API @@ -296,6 +386,151 @@ struct npy_api { } }; +template +struct is_complex : std::false_type {}; +template +struct is_complex> : std::true_type {}; + +template +struct npy_format_descriptor_name; + +template +struct npy_format_descriptor_name::value>> { + static constexpr auto name = const_name::value>( + const_name("numpy.bool"), + const_name::value>("numpy.int", "numpy.uint") + + const_name()); +}; + +template +struct npy_format_descriptor_name::value>> { + static constexpr auto name = const_name < std::is_same::value + || std::is_same::value + || std::is_same::value + || std::is_same::value + > (const_name("numpy.float") + const_name(), + const_name("numpy.longdouble")); +}; + +template +struct npy_format_descriptor_name::value>> { + static constexpr auto name = const_name < std::is_same::value + || std::is_same::value + || std::is_same::value + || std::is_same::value + > (const_name("numpy.complex") + + const_name(), + const_name("numpy.clongdouble")); +}; + +template +struct numpy_scalar_info {}; + +#define PYBIND11_NUMPY_SCALAR_IMPL(ctype_, typenum_) \ + template <> \ + struct numpy_scalar_info { \ + static constexpr auto name = npy_format_descriptor_name::name; \ + static constexpr int typenum = npy_api::typenum_##_; \ + } + +// boolean type +PYBIND11_NUMPY_SCALAR_IMPL(bool, NPY_BOOL); + +// character types +PYBIND11_NUMPY_SCALAR_IMPL(char, NPY_CHAR); +PYBIND11_NUMPY_SCALAR_IMPL(signed char, NPY_BYTE); +PYBIND11_NUMPY_SCALAR_IMPL(unsigned char, NPY_UBYTE); + +// signed integer types +PYBIND11_NUMPY_SCALAR_IMPL(std::int16_t, NPY_INT16); +PYBIND11_NUMPY_SCALAR_IMPL(std::int32_t, NPY_INT32); +PYBIND11_NUMPY_SCALAR_IMPL(std::int64_t, NPY_INT64); + +// unsigned integer types +PYBIND11_NUMPY_SCALAR_IMPL(std::uint16_t, NPY_UINT16); +PYBIND11_NUMPY_SCALAR_IMPL(std::uint32_t, NPY_UINT32); +PYBIND11_NUMPY_SCALAR_IMPL(std::uint64_t, NPY_UINT64); + +// floating point types +PYBIND11_NUMPY_SCALAR_IMPL(float, NPY_FLOAT); +PYBIND11_NUMPY_SCALAR_IMPL(double, NPY_DOUBLE); +PYBIND11_NUMPY_SCALAR_IMPL(long double, NPY_LONGDOUBLE); + +// complex types +PYBIND11_NUMPY_SCALAR_IMPL(std::complex, NPY_CFLOAT); +PYBIND11_NUMPY_SCALAR_IMPL(std::complex, NPY_CDOUBLE); +PYBIND11_NUMPY_SCALAR_IMPL(std::complex, NPY_CLONGDOUBLE); + +#undef PYBIND11_NUMPY_SCALAR_IMPL + +// This table normalizes typenums by mapping NPY_INT_, NPY_LONG, ... to NPY_INT32_, NPY_INT64, ... +// This is needed to correctly handle situations where multiple typenums map to the same type, +// e.g. NPY_LONG_ may be equivalent to NPY_INT_ or NPY_LONGLONG_ despite having a different +// typenum. The normalized typenum should always match the values used in npy_format_descriptor. +// If you change this code, please review `enum constants` above. +static constexpr int normalized_dtype_num[npy_api::NPY_VOID_ + 1] = { + // NPY_BOOL_ => + npy_api::NPY_BOOL_, + // NPY_BYTE_ => + npy_api::NPY_BYTE_, + // NPY_UBYTE_ => + npy_api::NPY_UBYTE_, + // NPY_SHORT_ => + npy_api::NPY_INT16_, + // NPY_USHORT_ => + npy_api::NPY_UINT16_, + // NPY_INT_ => + sizeof(int) == sizeof(std::int16_t) ? npy_api::NPY_INT16_ + : sizeof(int) == sizeof(std::int32_t) ? npy_api::NPY_INT32_ + : sizeof(int) == sizeof(std::int64_t) ? npy_api::NPY_INT64_ + : npy_api::NPY_INT_, + // NPY_UINT_ => + sizeof(unsigned int) == sizeof(std::uint16_t) ? npy_api::NPY_UINT16_ + : sizeof(unsigned int) == sizeof(std::uint32_t) ? npy_api::NPY_UINT32_ + : sizeof(unsigned int) == sizeof(std::uint64_t) ? npy_api::NPY_UINT64_ + : npy_api::NPY_UINT_, + // NPY_LONG_ => + sizeof(long) == sizeof(std::int16_t) ? npy_api::NPY_INT16_ + : sizeof(long) == sizeof(std::int32_t) ? npy_api::NPY_INT32_ + : sizeof(long) == sizeof(std::int64_t) ? npy_api::NPY_INT64_ + : npy_api::NPY_LONG_, + // NPY_ULONG_ => + sizeof(unsigned long) == sizeof(std::uint16_t) ? npy_api::NPY_UINT16_ + : sizeof(unsigned long) == sizeof(std::uint32_t) ? npy_api::NPY_UINT32_ + : sizeof(unsigned long) == sizeof(std::uint64_t) ? npy_api::NPY_UINT64_ + : npy_api::NPY_ULONG_, + // NPY_LONGLONG_ => + sizeof(long long) == sizeof(std::int16_t) ? npy_api::NPY_INT16_ + : sizeof(long long) == sizeof(std::int32_t) ? npy_api::NPY_INT32_ + : sizeof(long long) == sizeof(std::int64_t) ? npy_api::NPY_INT64_ + : npy_api::NPY_LONGLONG_, + // NPY_ULONGLONG_ => + sizeof(unsigned long long) == sizeof(std::uint16_t) ? npy_api::NPY_UINT16_ + : sizeof(unsigned long long) == sizeof(std::uint32_t) ? npy_api::NPY_UINT32_ + : sizeof(unsigned long long) == sizeof(std::uint64_t) ? npy_api::NPY_UINT64_ + : npy_api::NPY_ULONGLONG_, + // NPY_FLOAT_ => + npy_api::NPY_FLOAT_, + // NPY_DOUBLE_ => + npy_api::NPY_DOUBLE_, + // NPY_LONGDOUBLE_ => + npy_api::NPY_LONGDOUBLE_, + // NPY_CFLOAT_ => + npy_api::NPY_CFLOAT_, + // NPY_CDOUBLE_ => + npy_api::NPY_CDOUBLE_, + // NPY_CLONGDOUBLE_ => + npy_api::NPY_CLONGDOUBLE_, + // NPY_OBJECT_ => + npy_api::NPY_OBJECT_, + // NPY_STRING_ => + npy_api::NPY_STRING_, + // NPY_UNICODE_ => + npy_api::NPY_UNICODE_, + // NPY_VOID_ => + npy_api::NPY_VOID_, +}; + inline PyArray_Proxy *array_proxy(void *ptr) { return reinterpret_cast(ptr); } inline const PyArray_Proxy *array_proxy(const void *ptr) { @@ -310,6 +545,14 @@ inline const PyArrayDescr_Proxy *array_descriptor_proxy(const PyObject *ptr) { return reinterpret_cast(ptr); } +inline const PyArrayDescr1_Proxy *array_descriptor1_proxy(const PyObject *ptr) { + return reinterpret_cast(ptr); +} + +inline const PyArrayDescr2_Proxy *array_descriptor2_proxy(const PyObject *ptr) { + return reinterpret_cast(ptr); +} + inline bool check_flags(const void *ptr, int flag) { return (flag == (array_proxy(ptr)->flags & flag)); } @@ -318,10 +561,6 @@ template struct is_std_array : std::false_type {}; template struct is_std_array> : std::true_type {}; -template -struct is_complex : std::false_type {}; -template -struct is_complex> : std::true_type {}; template struct array_info_scalar { @@ -350,7 +589,7 @@ struct array_info> { } static constexpr auto extents = const_name::is_array>( - concat(const_name(), array_info::extents), const_name()); + ::pybind11::detail::concat(const_name(), array_info::extents), const_name()); }; // For numpy we have special handling for arrays of characters, so we don't include // the size in the array extents. @@ -535,8 +774,65 @@ template struct type_caster> : type_caster> {}; +template +struct type_caster> { + using value_type = T; + using type_info = numpy_scalar_info; + + PYBIND11_TYPE_CASTER(numpy_scalar, type_info::name); + + static handle &target_type() { + static handle tp = npy_api::get().PyArray_TypeObjectFromType_(type_info::typenum); + return tp; + } + + static handle &target_dtype() { + static handle tp = npy_api::get().PyArray_DescrFromType_(type_info::typenum); + return tp; + } + + bool load(handle src, bool) { + if (isinstance(src, target_type())) { + npy_api::get().PyArray_ScalarAsCtype_(src.ptr(), &value.value); + return true; + } + return false; + } + + static handle cast(numpy_scalar src, return_value_policy, handle) { + return npy_api::get().PyArray_Scalar_(&src.value, target_dtype().ptr(), nullptr); + } +}; + PYBIND11_NAMESPACE_END(detail) +template +struct numpy_scalar { + using value_type = T; + + value_type value; + + numpy_scalar() = default; + explicit numpy_scalar(value_type value) : value(value) {} + + explicit operator value_type() const { return value; } + numpy_scalar &operator=(value_type value) { + this->value = value; + return *this; + } + + friend bool operator==(const numpy_scalar &a, const numpy_scalar &b) { + return a.value == b.value; + } + + friend bool operator!=(const numpy_scalar &a, const numpy_scalar &b) { return !(a == b); } +}; + +template +numpy_scalar make_scalar(T value) { + return numpy_scalar(value); +} + class dtype : public object { public: PYBIND11_OBJECT_DEFAULT(dtype, object, detail::npy_api::get().PyArrayDescr_Check_) @@ -564,6 +860,8 @@ class dtype : public object { m_ptr = from_args(args).release().ptr(); } + /// Return dtype for the given typenum (one of the NPY_TYPES). + /// https://numpy.org/devdocs/reference/c-api/array.html#c.PyArray_DescrFromType explicit dtype(int typenum) : object(detail::npy_api::get().PyArray_DescrFromType_(typenum), stolen_t{}) { if (m_ptr == nullptr) { @@ -586,11 +884,32 @@ class dtype : public object { return detail::npy_format_descriptor::type>::dtype(); } + /// Return the type number associated with a C++ type. + /// This is the constexpr equivalent of `dtype::of().num()`. + template + static constexpr int num_of() { + return detail::npy_format_descriptor::type>::value; + } + /// Size of the data type in bytes. - ssize_t itemsize() const { return detail::array_descriptor_proxy(m_ptr)->elsize; } + ssize_t itemsize() const { + if (detail::npy_api::get().PyArray_RUNTIME_VERSION_ < 0x12) { + return detail::array_descriptor1_proxy(m_ptr)->elsize; + } + return detail::array_descriptor2_proxy(m_ptr)->elsize; + } /// Returns true for structured data types. - bool has_fields() const { return detail::array_descriptor_proxy(m_ptr)->names != nullptr; } + bool has_fields() const { + if (detail::npy_api::get().PyArray_RUNTIME_VERSION_ < 0x12) { + return detail::array_descriptor1_proxy(m_ptr)->names != nullptr; + } + const auto *proxy = detail::array_descriptor2_proxy(m_ptr); + if (proxy->type_num < 0 || proxy->type_num >= 2056) { + return false; + } + return proxy->names != nullptr; + } /// Single-character code for dtype's kind. /// For example, floating point types are 'f' and integral types are 'i'. @@ -605,7 +924,9 @@ class dtype : public object { return detail::array_descriptor_proxy(m_ptr)->type; } - /// type number of dtype. + /// Type number of dtype. Note that different values may be returned for equivalent types, + /// e.g. even though ``long`` may be equivalent to ``int`` or ``long long``, they still have + /// different type numbers. Consider using `normalized_num` to avoid this. int num() const { // Note: The signature, `dtype::num` follows the naming of NumPy's public // Python API (i.e., ``dtype.num``), rather than its internal @@ -613,23 +934,45 @@ class dtype : public object { return detail::array_descriptor_proxy(m_ptr)->type_num; } + /// Type number of dtype, normalized to match the return value of `num_of` for equivalent + /// types. This function can be used to write switch statements that correctly handle + /// equivalent types with different type numbers. + int normalized_num() const { + int value = num(); + if (value >= 0 && value <= detail::npy_api::NPY_VOID_) { + return detail::normalized_dtype_num[value]; + } + return value; + } + /// Single character for byteorder char byteorder() const { return detail::array_descriptor_proxy(m_ptr)->byteorder; } /// Alignment of the data type - int alignment() const { return detail::array_descriptor_proxy(m_ptr)->alignment; } + ssize_t alignment() const { + if (detail::npy_api::get().PyArray_RUNTIME_VERSION_ < 0x12) { + return detail::array_descriptor1_proxy(m_ptr)->alignment; + } + return detail::array_descriptor2_proxy(m_ptr)->alignment; + } /// Flags for the array descriptor - char flags() const { return detail::array_descriptor_proxy(m_ptr)->flags; } + std::uint64_t flags() const { + if (detail::npy_api::get().PyArray_RUNTIME_VERSION_ < 0x12) { + return (unsigned char) detail::array_descriptor1_proxy(m_ptr)->flags; + } + return detail::array_descriptor2_proxy(m_ptr)->flags; + } private: - static object _dtype_from_pep3118() { - static PyObject *obj = module_::import("numpy.core._internal") - .attr("_dtype_from_pep3118") - .cast() - .release() - .ptr(); - return reinterpret_borrow(obj); + static object &_dtype_from_pep3118() { + PYBIND11_CONSTINIT static gil_safe_call_once_and_store storage; + return storage + .call_once_and_store_result([]() { + return detail::import_numpy_core_submodule("_internal") + .attr("_dtype_from_pep3118"); + }) + .get_stored(); } dtype strip_padding(ssize_t itemsize) { @@ -762,7 +1105,11 @@ class array : public buffer { template array(ShapeContainer shape, StridesContainer strides, const T *ptr, handle base = handle()) - : array(pybind11::dtype::of(), std::move(shape), std::move(strides), ptr, base) {} + : array(pybind11::dtype::of(), + std::move(shape), + std::move(strides), + reinterpret_cast(ptr), + base) {} template array(ShapeContainer shape, const T *ptr, handle base = handle()) @@ -786,9 +1133,7 @@ class array : public buffer { } /// Byte size of a single element - ssize_t itemsize() const { - return detail::array_descriptor_proxy(detail::array_proxy(m_ptr)->descr)->elsize; - } + ssize_t itemsize() const { return dtype().itemsize(); } /// Total number of bytes ssize_t nbytes() const { return size() * itemsize(); } @@ -802,6 +1147,13 @@ class array : public buffer { /// Dimensions of the array const ssize_t *shape() const { return detail::array_proxy(m_ptr)->dimensions; } +#ifdef PYBIND11_HAS_SPAN + /// Dimensions of the array as a span + std::span shape_span() const { + return std::span(shape(), static_cast(ndim())); + } +#endif + /// Dimension along a given axis ssize_t shape(ssize_t dim) const { if (dim >= ndim()) { @@ -813,6 +1165,13 @@ class array : public buffer { /// Strides of the array const ssize_t *strides() const { return detail::array_proxy(m_ptr)->strides; } +#ifdef PYBIND11_HAS_SPAN + /// Strides of the array as a span + std::span strides_span() const { + return std::span(strides(), static_cast(ndim())); + } +#endif + /// Stride along a given axis ssize_t strides(ssize_t dim) const { if (dim >= ndim()) { @@ -1006,7 +1365,7 @@ class array : public buffer { /// Create array from any object -- always returns a new reference static PyObject *raw_array(PyObject *ptr, int ExtraFlags = 0) { if (ptr == nullptr) { - PyErr_SetString(PyExc_ValueError, "cannot create a pybind11::array from a nullptr"); + set_error(PyExc_ValueError, "cannot create a pybind11::array from a nullptr"); return nullptr; } return detail::npy_api::get().PyArray_FromAny_( @@ -1121,10 +1480,10 @@ class array_t : public array { /** * Returns a proxy object that provides const access to the array's data without bounds or - * dimensionality checking. Unlike `unchecked()`, this does not require that the underlying - * array have the `writable` flag. Use with care: the array must not be destroyed or reshaped - * for the duration of the returned object, and the caller must take care not to access invalid - * dimensions or dimension indices. + * dimensionality checking. Unlike `mutable_unchecked()`, this does not require that the + * underlying array have the `writable` flag. Use with care: the array must not be destroyed + * or reshaped for the duration of the returned object, and the caller must take care not to + * access invalid dimensions or dimension indices. */ template detail::unchecked_reference unchecked() const & { @@ -1153,7 +1512,7 @@ class array_t : public array { /// Create array from any object -- always returns a new reference static PyObject *raw_array_t(PyObject *ptr) { if (ptr == nullptr) { - PyErr_SetString(PyExc_ValueError, "cannot create a pybind11::array_t from a nullptr"); + set_error(PyExc_ValueError, "cannot create a pybind11::array_t from a nullptr"); return nullptr; } return detail::npy_api::get().PyArray_FromAny_(ptr, @@ -1225,38 +1584,6 @@ struct compare_buffer_info::valu } }; -template -struct npy_format_descriptor_name; - -template -struct npy_format_descriptor_name::value>> { - static constexpr auto name = const_name::value>( - const_name("bool"), - const_name::value>("numpy.int", "numpy.uint") - + const_name()); -}; - -template -struct npy_format_descriptor_name::value>> { - static constexpr auto name = const_name < std::is_same::value - || std::is_same::value - || std::is_same::value - || std::is_same::value - > (const_name("numpy.float") + const_name(), - const_name("numpy.longdouble")); -}; - -template -struct npy_format_descriptor_name::value>> { - static constexpr auto name = const_name < std::is_same::value - || std::is_same::value - || std::is_same::value - || std::is_same::value - > (const_name("numpy.complex") - + const_name(), - const_name("numpy.longcomplex")); -}; - template struct npy_format_descriptor< T, @@ -1283,12 +1610,20 @@ struct npy_format_descriptor< public: static constexpr int value = values[detail::is_fmt_numeric::index]; - static pybind11::dtype dtype() { - if (auto *ptr = npy_api::get().PyArray_DescrFromType_(value)) { - return reinterpret_steal(ptr); - } - pybind11_fail("Unsupported buffer format!"); - } + static pybind11::dtype dtype() { return pybind11::dtype(/*typenum*/ value); } +}; + +template +struct npy_format_descriptor< + T, + enable_if_t::value + || ((std::is_same::value || std::is_same::value) + && sizeof(T) == sizeof(PyObject *))>> { + static constexpr auto name = const_name("numpy.object_"); + + static constexpr int value = npy_api::NPY_OBJECT_; + + static pybind11::dtype dtype() { return pybind11::dtype(/*typenum*/ value); } }; #define PYBIND11_DECL_CHAR_FMT \ @@ -1412,7 +1747,9 @@ PYBIND11_NOINLINE void register_structured_dtype(any_container auto tindex = std::type_index(tinfo); numpy_internals.registered_dtypes[tindex] = {dtype_ptr, std::move(format_str)}; - get_internals().direct_conversions[tindex].push_back(direct_converter); + with_internals([tindex, &direct_converter](internals &internals) { + internals.direct_conversions[tindex].push_back(direct_converter); + }); } template @@ -1541,7 +1878,7 @@ class common_iterator { using value_type = container_type::value_type; using size_type = container_type::size_type; - common_iterator() : m_strides() {} + common_iterator() = default; common_iterator(void *ptr, const container_type &strides, const container_type &shape) : p_ptr(reinterpret_cast(ptr)), m_strides(strides.size()) { @@ -1843,7 +2180,7 @@ struct vectorize_helper { // Pointers to values the function was called with; the vectorized ones set here will start // out as array_t pointers, but they will be changed them to T pointers before we make // call the wrapped function. Non-vectorized pointers are left as-is. - std::array params{{&args...}}; + std::array params{{reinterpret_cast(&args)...}}; // The array of `buffer_info`s of vectorized arguments: std::array buffers{ @@ -1943,7 +2280,8 @@ vectorize_helper vectorize_extractor(const Func &f, Retur template struct handle_type_name> { static constexpr auto name - = const_name("numpy.ndarray[") + npy_format_descriptor::name + const_name("]"); + = io_name("typing.Annotated[numpy.typing.ArrayLike, ", "numpy.typing.NDArray[") + + npy_format_descriptor::name + const_name("]"); }; PYBIND11_NAMESPACE_END(detail) @@ -1989,4 +2327,86 @@ Helper vectorize(Return (Class::*f)(Args...) const) { return Helper(std::mem_fn(f)); } +// Intentionally no &&/const&& overloads: vectorized method calls operate on the bound Python +// instance and should not consume/move-from self. +// Vectorize a class method (non-const, lvalue ref-qualified): +template ())), + Return, + Class *, + Args...>> +Helper vectorize(Return (Class::*f)(Args...) &) { + return Helper(std::mem_fn(f)); +} + +// Vectorize a class method (const, lvalue ref-qualified): +template ())), + Return, + const Class *, + Args...>> +Helper vectorize(Return (Class::*f)(Args...) const &) { + return Helper(std::mem_fn(f)); +} + +#ifdef __cpp_noexcept_function_type +// Vectorize a class method (non-const, noexcept): +template ())), + Return, + Class *, + Args...>> +Helper vectorize(Return (Class::*f)(Args...) noexcept) { + return Helper(std::mem_fn(f)); +} + +// Vectorize a class method (const, noexcept): +template ())), + Return, + const Class *, + Args...>> +Helper vectorize(Return (Class::*f)(Args...) const noexcept) { + return Helper(std::mem_fn(f)); +} + +// Vectorize a class method (non-const, lvalue ref-qualified, noexcept): +template ())), + Return, + Class *, + Args...>> +Helper vectorize(Return (Class::*f)(Args...) & noexcept) { + return Helper(std::mem_fn(f)); +} + +// Vectorize a class method (const, lvalue ref-qualified, noexcept): +template ())), + Return, + const Class *, + Args...>> +Helper vectorize(Return (Class::*f)(Args...) const & noexcept) { + return Helper(std::mem_fn(f)); +} +#endif + PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/pybind11.h b/lib/pybind11/include/pybind11/pybind11.h index 6205eff..eebb130 100644 --- a/lib/pybind11/include/pybind11/pybind11.h +++ b/lib/pybind11/include/pybind11/pybind11.h @@ -9,32 +9,47 @@ */ #pragma once - #include "detail/class.h" +#include "detail/dynamic_raw_ptr_cast_if_possible.h" +#include "detail/exception_translation.h" +#include "detail/function_record_pyobject.h" +#include "detail/function_ref.h" #include "detail/init.h" +#include "detail/native_enum_data.h" +#include "detail/using_smart_holder.h" #include "attr.h" #include "gil.h" +#include "gil_safe_call_once.h" #include "options.h" +#include "trampoline_self_life_support.h" +#include "typing.h" +#include #include #include #include #include +#include #include #include #include -#if defined(__cpp_lib_launder) && !(defined(_MSC_VER) && (_MSC_VER < 1914)) -# define PYBIND11_STD_LAUNDER std::launder -# define PYBIND11_HAS_STD_LAUNDER 1 -#else -# define PYBIND11_STD_LAUNDER -# define PYBIND11_HAS_STD_LAUNDER 0 +// See PR #5448. This warning suppression is needed for the PYBIND11_OVERRIDE macro family. +// NOTE that this is NOT embedded in a push/pop pair because that is very difficult to achieve. +#if defined(__clang_major__) && __clang_major__ < 14 +PYBIND11_WARNING_DISABLE_CLANG("-Wgnu-zero-variadic-macro-arguments") #endif + #if defined(__GNUG__) && !defined(__clang__) # include #endif +#if defined(__cpp_if_constexpr) && __cpp_if_constexpr >= 201606 +# define PYBIND11_MAYBE_CONSTEXPR constexpr +#else +# define PYBIND11_MAYBE_CONSTEXPR +#endif + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) /* https://stackoverflow.com/questions/46798456/handling-gccs-noexcept-type-warning @@ -52,22 +67,193 @@ PYBIND11_WARNING_DISABLE_MSVC(4127) PYBIND11_NAMESPACE_BEGIN(detail) -// Apply all the extensions translators from a list -// Return true if one of the translators completed without raising an exception -// itself. Return of false indicates that if there are other translators -// available, they should be tried. -inline bool apply_exception_translators(std::forward_list &translators) { - auto last_exception = std::current_exception(); +inline std::string replace_newlines_and_squash(const char *text) { + const char *whitespaces = " \t\n\r\f\v"; + std::string result(text); + bool previous_is_whitespace = false; - for (auto &translator : translators) { - try { - translator(last_exception); - return true; - } catch (...) { - last_exception = std::current_exception(); + if (result.size() >= 2) { + // Do not modify string representations + char first_char = result[0]; + char last_char = result[result.size() - 1]; + if (first_char == last_char && first_char == '\'') { + return result; + } + } + result.clear(); + + // Replace characters in whitespaces array with spaces and squash consecutive spaces + while (*text != '\0') { + if (std::strchr(whitespaces, *text)) { + if (!previous_is_whitespace) { + result += ' '; + previous_is_whitespace = true; + } + } else { + result += *text; + previous_is_whitespace = false; + } + ++text; + } + + // Strip leading and trailing whitespaces + const size_t str_begin = result.find_first_not_of(whitespaces); + if (str_begin == std::string::npos) { + return ""; + } + + const size_t str_end = result.find_last_not_of(whitespaces); + const size_t str_range = str_end - str_begin + 1; + + return result.substr(str_begin, str_range); +} + +/* Generate a proper function signature */ +inline std::string generate_function_signature(const char *type_caster_name_field, + detail::function_record *func_rec, + const std::type_info *const *types, + size_t &type_index, + size_t &arg_index) { + std::string signature; + bool is_starred = false; + // `is_return_value.top()` is true if we are currently inside the return type of the + // signature. Using `@^`/`@$` we can force types to be arg/return types while `@!` pops + // back to the previous state. + std::stack is_return_value({false}); + // The following characters have special meaning in the signature parsing. Literals + // containing these are escaped with `!`. + std::string special_chars("!@%{}-"); + for (const auto *pc = type_caster_name_field; *pc != '\0'; ++pc) { + const auto c = *pc; + if (c == '{') { + // Write arg name for everything except *args and **kwargs. + // Detect {@*args...} or {@**kwargs...} + is_starred = *(pc + 1) == '@' && *(pc + 2) == '*'; + if (is_starred) { + continue; + } + // Separator for keyword-only arguments, placed before the kw + // arguments start (unless we are already putting an *args) + if (!func_rec->has_args && arg_index == func_rec->nargs_pos) { + signature += "*, "; + } + if (arg_index < func_rec->args.size() && func_rec->args[arg_index].name) { + signature += func_rec->args[arg_index].name; + } else if (arg_index == 0 && func_rec->is_method) { + signature += "self"; + } else { + signature += "arg" + std::to_string(arg_index - (func_rec->is_method ? 1 : 0)); + } + signature += ": "; + } else if (c == '}') { + // Write default value if available. + if (!is_starred && arg_index < func_rec->args.size() + && func_rec->args[arg_index].descr) { + signature += " = "; + signature += detail::replace_newlines_and_squash(func_rec->args[arg_index].descr); + } + // Separator for positional-only arguments (placed after the + // argument, rather than before like * + if (func_rec->nargs_pos_only > 0 && (arg_index + 1) == func_rec->nargs_pos_only) { + signature += ", /"; + } + if (!is_starred) { + arg_index++; + } + } else if (c == '%') { + const std::type_info *t = types[type_index++]; + if (!t) { + pybind11_fail("Internal error while parsing type signature (1)"); + } + if (auto *tinfo = detail::get_type_info(*t)) { + handle th(reinterpret_cast(tinfo->type)); + signature += th.attr("__module__").cast() + "." + + th.attr("__qualname__").cast(); + } else if (auto th = detail::global_internals_native_enum_type_map_get_item(*t)) { + signature += th.attr("__module__").cast() + "." + + th.attr("__qualname__").cast(); + } else if (func_rec->is_new_style_constructor && arg_index == 0) { + // A new-style `__init__` takes `self` as `value_and_holder`. + // Rewrite it to the proper class type. + signature += func_rec->scope.attr("__module__").cast() + "." + + func_rec->scope.attr("__qualname__").cast(); + } else { + signature += detail::quote_cpp_type_name(detail::clean_type_id(t->name())); + } + } else if (c == '!' && special_chars.find(*(pc + 1)) != std::string::npos) { + // typing::Literal escapes special characters with ! + signature += *++pc; + } else if (c == '@') { + // `@^ ... @!` and `@$ ... @!` are used to force arg/return value type (see + // typing::Callable/detail::arg_descr/detail::return_descr). + // `@~ ... @!` inverts the current context (see detail::inv_descr). + if (*(pc + 1) == '^') { + is_return_value.emplace(false); + ++pc; + continue; + } + if (*(pc + 1) == '$') { + is_return_value.emplace(true); + ++pc; + continue; + } + if (*(pc + 1) == '~') { + is_return_value.emplace(!is_return_value.top()); + ++pc; + continue; + } + if (*(pc + 1) == '!') { + is_return_value.pop(); + ++pc; + continue; + } + // Handle types that differ depending on whether they appear + // in an argument or a return value position (see io_name). + // For named arguments (py::arg()) with noconvert set, return value type is used. + ++pc; + if (!is_return_value.top() + && (!(arg_index < func_rec->args.size() && !func_rec->args[arg_index].convert))) { + while (*pc != '\0' && *pc != '@') { + signature += *pc++; + } + if (*pc == '@') { + ++pc; + } + while (*pc != '\0' && *pc != '@') { + ++pc; + } + } else { + while (*pc != '\0' && *pc != '@') { + ++pc; + } + if (*pc == '@') { + ++pc; + } + while (*pc != '\0' && *pc != '@') { + signature += *pc++; + } + } + } else { + if (c == '-' && *(pc + 1) == '>') { + is_return_value.emplace(true); + } + signature += c; } } - return false; + return signature; +} + +template +inline std::string generate_type_signature() { + static constexpr auto caster_name_field = make_caster::name; + PYBIND11_DESCR_CONSTEXPR auto descr_types = decltype(caster_name_field)::types(); + // Create a default function_record to ensure the function signature has the proper + // configuration e.g. no_convert. + auto func_rec = function_record(); + size_t type_index = 0; + size_t arg_index = 0; + return generate_function_signature( + caster_name_field.text, &func_rec, descr_types.data(), type_index, arg_index); } #if defined(_MSC_VER) @@ -76,6 +262,49 @@ inline bool apply_exception_translators(std::forward_list & # define PYBIND11_COMPAT_STRDUP strdup #endif +#define PYBIND11_READABLE_FUNCTION_SIGNATURE_EXPR \ + detail::const_name("(") + cast_in::arg_names + detail::const_name(") -> ") + cast_out::name + +// We factor out readable function signatures to a specific template +// so that they don't get duplicated across different instantiations of +// cpp_function::initialize (which is templated on more types). +template +class ReadableFunctionSignature { +public: + using sig_type = decltype(PYBIND11_READABLE_FUNCTION_SIGNATURE_EXPR); + +private: + // We have to repeat PYBIND11_READABLE_FUNCTION_SIGNATURE_EXPR in decltype() + // because C++11 doesn't allow functions to return `auto`. (We don't + // know the type because it's some variant of detail::descr with + // unknown N.) + static constexpr sig_type sig() { return PYBIND11_READABLE_FUNCTION_SIGNATURE_EXPR; } + +public: + static constexpr sig_type kSig = sig(); + // We can only stash the result of detail::descr::types() in a + // constexpr variable if we aren't on MSVC (see + // PYBIND11_DESCR_CONSTEXPR). +#if !defined(_MSC_VER) + using types_type = decltype(sig_type::types()); + static constexpr types_type kTypes = sig_type::types(); +#endif +}; +#undef PYBIND11_READABLE_FUNCTION_SIGNATURE_EXPR + +// Prior to C++17, we don't have inline variables, so we have to +// provide an out-of-line definition of the class member. +#if !defined(PYBIND11_CPP17) +template +constexpr typename ReadableFunctionSignature::sig_type + ReadableFunctionSignature::kSig; +# if !defined(_MSC_VER) +template +constexpr typename ReadableFunctionSignature::types_type + ReadableFunctionSignature::kTypes; +# endif +#endif + PYBIND11_NAMESPACE_END(detail) /// Wraps an arbitrary C++ function/method/lambda function/.. into a callable Python object @@ -84,6 +313,7 @@ class cpp_function : public function { cpp_function() = default; // NOLINTNEXTLINE(google-explicit-constructor) cpp_function(std::nullptr_t) {} + cpp_function(std::nullptr_t, const is_setter &) {} /// Construct a cpp_function from a vanilla function pointer template @@ -108,7 +338,7 @@ class cpp_function : public function { cpp_function(Return (Class::*f)(Arg...), const Extra &...extra) { initialize( [f](Class *c, Arg... args) -> Return { return (c->*f)(std::forward(args)...); }, - (Return(*)(Class *, Arg...)) nullptr, + (Return (*)(Class *, Arg...)) nullptr, extra...); } @@ -120,7 +350,7 @@ class cpp_function : public function { cpp_function(Return (Class::*f)(Arg...) &, const Extra &...extra) { initialize( [f](Class *c, Arg... args) -> Return { return (c->*f)(std::forward(args)...); }, - (Return(*)(Class *, Arg...)) nullptr, + (Return (*)(Class *, Arg...)) nullptr, extra...); } @@ -130,7 +360,7 @@ class cpp_function : public function { cpp_function(Return (Class::*f)(Arg...) const, const Extra &...extra) { initialize([f](const Class *c, Arg... args) -> Return { return (c->*f)(std::forward(args)...); }, - (Return(*)(const Class *, Arg...)) nullptr, + (Return (*)(const Class *, Arg...)) nullptr, extra...); } @@ -142,10 +372,100 @@ class cpp_function : public function { cpp_function(Return (Class::*f)(Arg...) const &, const Extra &...extra) { initialize([f](const Class *c, Arg... args) -> Return { return (c->*f)(std::forward(args)...); }, - (Return(*)(const Class *, Arg...)) nullptr, + (Return (*)(const Class *, Arg...)) nullptr, + extra...); + } + + /// Construct a cpp_function from a class method (non-const, rvalue ref-qualifier) + template + // NOLINTNEXTLINE(google-explicit-constructor) + cpp_function(Return (Class::*f)(Arg...) &&, const Extra &...extra) { + initialize( + [f](Class *c, Arg... args) -> Return { + return (std::move(*c).*f)(std::forward(args)...); + }, + (Return (*)(Class *, Arg...)) nullptr, + extra...); + } + + /// Construct a cpp_function from a class method (const, rvalue ref-qualifier) + template + // NOLINTNEXTLINE(google-explicit-constructor) + cpp_function(Return (Class::*f)(Arg...) const &&, const Extra &...extra) { + initialize( + [f](const Class *c, Arg... args) -> Return { + return (std::move(*c).*f)(std::forward(args)...); + }, + (Return (*)(const Class *, Arg...)) nullptr, + extra...); + } + +#ifdef __cpp_noexcept_function_type + /// Construct a cpp_function from a class method (non-const, no ref-qualifier, noexcept) + template + // NOLINTNEXTLINE(google-explicit-constructor) + cpp_function(Return (Class::*f)(Arg...) noexcept, const Extra &...extra) { + initialize( + [f](Class *c, Arg... args) -> Return { return (c->*f)(std::forward(args)...); }, + (Return (*)(Class *, Arg...)) nullptr, + extra...); + } + + /// Construct a cpp_function from a class method (non-const, lvalue ref-qualifier, noexcept) + template + // NOLINTNEXTLINE(google-explicit-constructor) + cpp_function(Return (Class::*f)(Arg...) & noexcept, const Extra &...extra) { + initialize( + [f](Class *c, Arg... args) -> Return { return (c->*f)(std::forward(args)...); }, + (Return (*)(Class *, Arg...)) nullptr, + extra...); + } + + /// Construct a cpp_function from a class method (const, no ref-qualifier, noexcept) + template + // NOLINTNEXTLINE(google-explicit-constructor) + cpp_function(Return (Class::*f)(Arg...) const noexcept, const Extra &...extra) { + initialize([f](const Class *c, + Arg... args) -> Return { return (c->*f)(std::forward(args)...); }, + (Return (*)(const Class *, Arg...)) nullptr, + extra...); + } + + /// Construct a cpp_function from a class method (const, lvalue ref-qualifier, noexcept) + template + // NOLINTNEXTLINE(google-explicit-constructor) + cpp_function(Return (Class::*f)(Arg...) const & noexcept, const Extra &...extra) { + initialize([f](const Class *c, + Arg... args) -> Return { return (c->*f)(std::forward(args)...); }, + (Return (*)(const Class *, Arg...)) nullptr, extra...); } + /// Construct a cpp_function from a class method (non-const, rvalue ref-qualifier, noexcept) + template + // NOLINTNEXTLINE(google-explicit-constructor) + cpp_function(Return (Class::*f)(Arg...) && noexcept, const Extra &...extra) { + initialize( + [f](Class *c, Arg... args) -> Return { + return (std::move(*c).*f)(std::forward(args)...); + }, + (Return (*)(Class *, Arg...)) nullptr, + extra...); + } + + /// Construct a cpp_function from a class method (const, rvalue ref-qualifier, noexcept) + template + // NOLINTNEXTLINE(google-explicit-constructor) + cpp_function(Return (Class::*f)(Arg...) const && noexcept, const Extra &...extra) { + initialize( + [f](const Class *c, Arg... args) -> Return { + return (std::move(*c).*f)(std::forward(args)...); + }, + (Return (*)(const Class *, Arg...)) nullptr, + extra...); + } +#endif + /// Return the function name object name() const { return attr("__name__"); } @@ -163,12 +483,56 @@ class cpp_function : public function { return unique_function_record(new detail::function_record()); } +private: + // This is outlined from the dispatch lambda in initialize to save + // on code size. Crucially, we use function_ref to type-erase the + // actual function lambda so that we can get code reuse for + // functions with the same Return, Args, and Guard. + template + static handle call_impl(detail::function_call &call, detail::function_ref f) { + using namespace detail; + // Static assertion: function_ref must be trivially copyable to ensure safe pass-by-value. + // Lifetime safety: The function_ref is created from cap->f which lives in the capture + // object stored in the function record, and is only used synchronously within this + // function call. It is never stored beyond the scope of call_impl. + static_assert(std::is_trivially_copyable>::value, + "function_ref must be trivially copyable for safe pass-by-value usage"); + using cast_out + = make_caster::value, void_type, Return>>; + + ArgsConverter args_converter; + if (!args_converter.load_args(call)) { + return PYBIND11_TRY_NEXT_OVERLOAD; + } + + /* Override policy for rvalues -- usually to enforce rvp::move on an rvalue */ + return_value_policy policy + = return_value_policy_override::policy(call.func.policy); + + /* Perform the function call */ + handle result; + if (call.func.is_setter) { + (void) std::move(args_converter).template call(f); + result = none().release(); + } else { + result = cast_out::cast( + std::move(args_converter).template call(f), policy, call.parent); + } + + return result; + } + +protected: /// Special internal constructor for functors, lambda functions, etc. template void initialize(Func &&f, Return (*)(Args...), const Extra &...extra) { using namespace detail; struct capture { remove_reference_t f; + + static capture *from_data(void **data) { + return PYBIND11_STD_LAUNDER(reinterpret_cast(data)); + } }; /* Store the function including any extra state it might have (e.g. a lambda capture @@ -188,7 +552,7 @@ class cpp_function : public function { PYBIND11_WARNING_DISABLE_GCC("-Wplacement-new") #endif - new ((capture *) &rec->data) capture{std::forward(f)}; + new (capture::from_data(rec->data)) capture{std::forward(f)}; #if !PYBIND11_HAS_STD_LAUNDER PYBIND11_WARNING_DISABLE_GCC("-Wstrict-aliasing") @@ -198,8 +562,8 @@ class cpp_function : public function { // a significant refactoring it's "impossible" to solve. if (!std::is_trivially_destructible::value) { rec->free_data = [](function_record *r) { - auto data = PYBIND11_STD_LAUNDER((capture *) &r->data); - (void) data; + auto data = capture::from_data(r->data); + (void) data; // suppress "unused variable" warnings data->~capture(); }; } @@ -221,13 +585,6 @@ class cpp_function : public function { /* Dispatch code which converts function arguments and performs the actual function call */ rec->impl = [](function_call &call) -> handle { - cast_in args_converter; - - /* Try to cast the function arguments into the C++ domain */ - if (!args_converter.load_args(call)) { - return PYBIND11_TRY_NEXT_OVERLOAD; - } - /* Invoke call policy pre-call hook */ process_attributes::precall(call); @@ -236,18 +593,11 @@ class cpp_function : public function { : call.func.data[0]); auto *cap = const_cast(reinterpret_cast(data)); - /* Override policy for rvalues -- usually to enforce rvp::move on an rvalue */ - return_value_policy policy - = return_value_policy_override::policy(call.func.policy); - - /* Function scope guard -- defaults to the compile-to-nothing `void_type` */ - using Guard = extract_guard_t; - - /* Perform the function call */ - handle result - = cast_out::cast(std::move(args_converter).template call(cap->f), - policy, - call.parent); + auto result = call_impl, + cast_in>(call, detail::function_ref(cap->f)); /* Invoke call policy post-call hook */ process_attributes::postcall(call, result); @@ -269,9 +619,20 @@ class cpp_function : public function { constexpr bool has_kw_only_args = any_of...>::value, has_pos_only_args = any_of...>::value, has_arg_annotations = any_of...>::value; + constexpr bool has_is_method = any_of...>::value; + // The implicit `self` argument is not present and not counted in method definitions. + constexpr bool has_args = cast_in::args_pos >= 0; + constexpr bool is_method_with_self_arg_only = has_is_method && !has_args; static_assert(has_arg_annotations || !has_kw_only_args, "py::kw_only requires the use of argument annotations"); - static_assert(has_arg_annotations || !has_pos_only_args, + static_assert(((/* Need `py::arg("arg_name")` annotation in function/method. */ + has_arg_annotations) + || (/* Allow methods with no arguments `def method(self, /): ...`. + * A method has at least one argument `self`. There can be no + * `py::arg` annotation. E.g. `class.def("method", py::pos_only())`. + */ + is_method_with_self_arg_only)) + || !has_pos_only_args, "py::pos_only requires the use of argument annotations (for docstrings " "and aligning the annotations to the argument)"); @@ -287,9 +648,14 @@ class cpp_function : public function { /* Generate a readable signature describing the function's arguments and return value types */ - static constexpr auto signature - = const_name("(") + cast_in::arg_names + const_name(") -> ") + cast_out::name; - PYBIND11_DESCR_CONSTEXPR auto types = decltype(signature)::types(); + static constexpr const auto &signature + = detail::ReadableFunctionSignature::kSig; +#if !defined(_MSC_VER) + static constexpr const auto &types + = detail::ReadableFunctionSignature::kTypes; +#else + PYBIND11_DESCR_CONSTEXPR auto types = std::decay::type::types(); +#endif /* Register the function with Python from generic (non-templated) code */ // Pass on the ownership over the `unique_rec` to `initialize_generic`. `rec` stays valid. @@ -299,6 +665,8 @@ class cpp_function : public function { using FunctionType = Return (*)(Args...); constexpr bool is_function_ptr = std::is_convertible::value && sizeof(capture) == sizeof(void *); + PYBIND11_ENSURE_PRECONDITION_FOR_FUNCTIONAL_H_PERFORMANCE_OPTIMIZATIONS( + !is_function_ptr || std::is_standard_layout::value); if (is_function_ptr) { rec->is_stateless = true; rec->data[1] @@ -387,69 +755,9 @@ class cpp_function : public function { } #endif - /* Generate a proper function signature */ - std::string signature; size_t type_index = 0, arg_index = 0; - bool is_starred = false; - for (const auto *pc = text; *pc != '\0'; ++pc) { - const auto c = *pc; - - if (c == '{') { - // Write arg name for everything except *args and **kwargs. - is_starred = *(pc + 1) == '*'; - if (is_starred) { - continue; - } - // Separator for keyword-only arguments, placed before the kw - // arguments start (unless we are already putting an *args) - if (!rec->has_args && arg_index == rec->nargs_pos) { - signature += "*, "; - } - if (arg_index < rec->args.size() && rec->args[arg_index].name) { - signature += rec->args[arg_index].name; - } else if (arg_index == 0 && rec->is_method) { - signature += "self"; - } else { - signature += "arg" + std::to_string(arg_index - (rec->is_method ? 1 : 0)); - } - signature += ": "; - } else if (c == '}') { - // Write default value if available. - if (!is_starred && arg_index < rec->args.size() && rec->args[arg_index].descr) { - signature += " = "; - signature += rec->args[arg_index].descr; - } - // Separator for positional-only arguments (placed after the - // argument, rather than before like * - if (rec->nargs_pos_only > 0 && (arg_index + 1) == rec->nargs_pos_only) { - signature += ", /"; - } - if (!is_starred) { - arg_index++; - } - } else if (c == '%') { - const std::type_info *t = types[type_index++]; - if (!t) { - pybind11_fail("Internal error while parsing type signature (1)"); - } - if (auto *tinfo = detail::get_type_info(*t)) { - handle th((PyObject *) tinfo->type); - signature += th.attr("__module__").cast() + "." - + th.attr("__qualname__").cast(); - } else if (rec->is_new_style_constructor && arg_index == 0) { - // A new-style `__init__` takes `self` as `value_and_holder`. - // Rewrite it to the proper class type. - signature += rec->scope.attr("__module__").cast() + "." - + rec->scope.attr("__qualname__").cast(); - } else { - std::string tname(t->name()); - detail::clean_type_id(tname); - signature += tname; - } - } else { - signature += c; - } - } + std::string signature + = detail::generate_function_signature(text, rec, types, type_index, arg_index); if (arg_index != args - rec->has_args - rec->has_kwargs || types[type_index] != nullptr) { pybind11_fail("Internal error while parsing type signature (2)"); @@ -457,7 +765,7 @@ class cpp_function : public function { rec->signature = guarded_strdup(signature.c_str()); rec->args.shrink_to_fit(); - rec->nargs = (std::uint16_t) args; + rec->nargs = static_cast(args); if (rec->sibling && PYBIND11_INSTANCE_METHOD_CHECK(rec->sibling.ptr())) { rec->sibling = PYBIND11_INSTANCE_METHOD_GET_FUNCTION(rec->sibling.ptr()); @@ -467,20 +775,15 @@ class cpp_function : public function { if (rec->sibling) { if (PyCFunction_Check(rec->sibling.ptr())) { auto *self = PyCFunction_GET_SELF(rec->sibling.ptr()); - if (!isinstance(self)) { + if (self == nullptr) { + pybind11_fail( + "initialize_generic: Unexpected nullptr from PyCFunction_GET_SELF"); + } + chain = detail::function_record_ptr_from_PyObject(self); + if (chain && !chain->scope.is(rec->scope)) { + /* Never append a method to an overload chain of a parent class; + instead, hide the parent's overloads in this case */ chain = nullptr; - } else { - auto rec_capsule = reinterpret_borrow(self); - if (detail::is_function_record_capsule(rec_capsule)) { - chain = rec_capsule.get_pointer(); - /* Never append a method to an overload chain of a parent class; - instead, hide the parent's overloads in this case */ - if (!chain->scope.is(rec->scope)) { - chain = nullptr; - } - } else { - chain = nullptr; - } } } // Don't trigger for things like the default __init__, which are wrapper_descriptors @@ -498,23 +801,15 @@ class cpp_function : public function { rec->def->ml_name = rec->name; rec->def->ml_meth = reinterpret_cast(reinterpret_cast(dispatcher)); - rec->def->ml_flags = METH_VARARGS | METH_KEYWORDS; + rec->def->ml_flags = METH_FASTCALL | METH_KEYWORDS; - capsule rec_capsule(unique_rec.release(), - [](void *ptr) { destruct((detail::function_record *) ptr); }); - rec_capsule.set_name(detail::get_function_record_capsule_name()); + object py_func_rec = detail::function_record_PyObject_New(); + (reinterpret_cast(py_func_rec.ptr()))->cpp_func_rec + = unique_rec.release(); guarded_strdup.release(); - object scope_module; - if (rec->scope) { - if (hasattr(rec->scope, "__module__")) { - scope_module = rec->scope.attr("__module__"); - } else if (hasattr(rec->scope, "__name__")) { - scope_module = rec->scope.attr("__name__"); - } - } - - m_ptr = PyCFunction_NewEx(rec->def, rec_capsule.ptr(), scope_module.ptr()); + object scope_module = detail::get_scope_module(rec->scope); + m_ptr = PyCFunction_NewEx(rec->def, py_func_rec.ptr(), scope_module.ptr()); if (!m_ptr) { pybind11_fail("cpp_function::cpp_function(): Could not allocate function object"); } @@ -543,9 +838,9 @@ class cpp_function : public function { // chain. chain_start = rec; rec->next = chain; - auto rec_capsule - = reinterpret_borrow(((PyCFunctionObject *) m_ptr)->m_self); - rec_capsule.set_pointer(unique_rec.release()); + auto *py_func_rec = reinterpret_cast( + PyCFunction_GET_SELF(m_ptr)); + py_func_rec->cpp_func_rec = unique_rec.release(); guarded_strdup.release(); } else { // Or end of chain (normal behavior) @@ -562,7 +857,8 @@ class cpp_function : public function { int index = 0; /* Create a nice pydoc rec including all signatures and docstrings of the functions in the overload chain */ - if (chain && options::show_function_signatures()) { + if (chain && options::show_function_signatures() + && std::strcmp(rec->name, "_pybind11_conduit_v1_") != 0) { // First a generic signature signatures += rec->name; signatures += "(*args, **kwargs)\n"; @@ -571,7 +867,8 @@ class cpp_function : public function { // Then specific overload signatures bool first_user_def = true; for (auto *it = chain_start; it != nullptr; it = it->next) { - if (options::show_function_signatures()) { + if (options::show_function_signatures() + && std::strcmp(rec->name, "_pybind11_conduit_v1_") != 0) { if (index > 0) { signatures += '\n'; } @@ -602,12 +899,11 @@ class cpp_function : public function { } } - /* Install docstring */ - auto *func = (PyCFunctionObject *) m_ptr; - std::free(const_cast(func->m_ml->ml_doc)); + auto *func = reinterpret_cast(m_ptr); // Install docstring if it's non-empty (when at least one option is enabled) - func->m_ml->ml_doc - = signatures.empty() ? nullptr : PYBIND11_COMPAT_STRDUP(signatures.c_str()); + auto *doc = signatures.empty() ? nullptr : PYBIND11_COMPAT_STRDUP(signatures.c_str()); + std::free(const_cast(PYBIND11_PYCFUNCTION_GET_DOC(func))); + PYBIND11_PYCFUNCTION_SET_DOC(func, doc); if (rec->is_method) { m_ptr = PYBIND11_INSTANCE_METHOD_NEW(m_ptr, rec->scope.ptr()); @@ -619,6 +915,8 @@ class cpp_function : public function { } } + friend void detail::function_record_PyTypeObject_methods::tp_dealloc_impl(PyObject *); + /// When a cpp_function is GCed, release any memory allocated by pybind11 static void destruct(detail::function_record *rec, bool free_strings = true) { // If on Python 3.9, check the interpreter "MICRO" (patch) version. @@ -636,9 +934,9 @@ class cpp_function : public function { // so they cannot be freed. Once the function has been created, they can. // Check `make_function_record` for more details. if (free_strings) { - std::free((char *) rec->name); - std::free((char *) rec->doc); - std::free((char *) rec->signature); + std::free(rec->name); + std::free(rec->doc); + std::free(rec->signature); for (auto &arg : rec->args) { std::free(const_cast(arg.name)); std::free(const_cast(arg.descr)); @@ -666,34 +964,33 @@ class cpp_function : public function { } /// Main dispatch logic for calls to functions bound using pybind11 - static PyObject *dispatcher(PyObject *self, PyObject *args_in, PyObject *kwargs_in) { + static PyObject * + dispatcher(PyObject *self, PyObject *const *args_in_arr, size_t nargsf, PyObject *kwnames_in) { using namespace detail; - assert(isinstance(self)); + const function_record *overloads = function_record_ptr_from_PyObject(self); + assert(overloads != nullptr); /* Iterator over the list of potentially admissible overloads */ - const function_record *overloads = reinterpret_cast( - PyCapsule_GetPointer(self, get_function_record_capsule_name())), - *it = overloads; - assert(overloads != nullptr); + const function_record *current_overload = overloads; /* Need to know how many arguments + keyword arguments there are to pick the right overload */ - const auto n_args_in = (size_t) PyTuple_GET_SIZE(args_in); + const auto n_args_in = static_cast(PyVectorcall_NARGS(nargsf)); - handle parent = n_args_in > 0 ? PyTuple_GET_ITEM(args_in, 0) : nullptr, + handle parent = n_args_in > 0 ? args_in_arr[0] : nullptr, result = PYBIND11_TRY_NEXT_OVERLOAD; auto self_value_and_holder = value_and_holder(); if (overloads->is_constructor) { if (!parent || !PyObject_TypeCheck(parent.ptr(), (PyTypeObject *) overloads->scope.ptr())) { - PyErr_SetString( - PyExc_TypeError, - "__init__(self, ...) called with invalid or missing `self` argument"); + set_error(PyExc_TypeError, + "__init__(self, ...) called with invalid or missing `self` argument"); return nullptr; } - auto *const tinfo = get_type_info((PyTypeObject *) overloads->scope.ptr()); + auto *const tinfo + = get_type_info(reinterpret_cast(overloads->scope.ptr())); auto *const pi = reinterpret_cast(parent.ptr()); self_value_and_holder = pi->get_value_and_holder(tinfo, true); @@ -712,9 +1009,10 @@ class cpp_function : public function { std::vector second_pass; // However, if there are no overloads, we can just skip the no-convert pass entirely - const bool overloaded = it != nullptr && it->next != nullptr; + const bool overloaded + = current_overload != nullptr && current_overload->next != nullptr; - for (; it != nullptr; it = it->next) { + for (; current_overload != nullptr; current_overload = current_overload->next) { /* For each overload: 1. Copy all positional arguments we were given, also checking to make sure that @@ -735,7 +1033,7 @@ class cpp_function : public function { a result other than PYBIND11_TRY_NEXT_OVERLOAD. */ - const function_record &func = *it; + const function_record &func = *current_overload; size_t num_args = func.nargs; // Number of positional arguments that we need if (func.has_args) { --num_args; // (but don't count py::args @@ -757,7 +1055,7 @@ class cpp_function : public function { function_call call(func, parent); // Protect std::min with parentheses - size_t args_to_copy = (std::min)(pos_args, n_args_in); + size_t args_to_copy = (std::min) (pos_args, n_args_in); size_t args_copied = 0; // 0. Inject new-style `self` argument @@ -768,7 +1066,7 @@ class cpp_function : public function { self_value_and_holder.type->dealloc(self_value_and_holder); } - call.init_self = PyTuple_GET_ITEM(args_in, 0); + call.init_self = args_in_arr[0]; call.args.emplace_back(reinterpret_cast(&self_value_and_holder)); call.args_convert.push_back(false); ++args_copied; @@ -779,17 +1077,24 @@ class cpp_function : public function { for (; args_copied < args_to_copy; ++args_copied) { const argument_record *arg_rec = args_copied < func.args.size() ? &func.args[args_copied] : nullptr; - if (kwargs_in && arg_rec && arg_rec->name - && dict_getitemstring(kwargs_in, arg_rec->name)) { + + /* if the argument is listed in the call site's kwargs, but the argument is + also fulfilled positionally, then the call can't match this overload. for + example, the call site is: foo(0, key=1) but our overload is foo(key:int) then + this call can't be for us, because it would be invalid. + */ + if (kwnames_in && arg_rec && arg_rec->name + && keyword_index(kwnames_in, arg_rec->name) >= 0) { bad_arg = true; break; } - handle arg(PyTuple_GET_ITEM(args_in, args_copied)); + handle arg(args_in_arr[args_copied]); if (arg_rec && !arg_rec->none && arg.is_none()) { bad_arg = true; break; } + call.args.push_back(arg); call.args_convert.push_back(arg_rec ? arg_rec->convert : true); } @@ -801,20 +1106,12 @@ class cpp_function : public function { // to copy the rest into a py::args argument. size_t positional_args_copied = args_copied; - // We'll need to copy this if we steal some kwargs for defaults - dict kwargs = reinterpret_borrow(kwargs_in); - // 1.5. Fill in any missing pos_only args from defaults if they exist if (args_copied < func.nargs_pos_only) { for (; args_copied < func.nargs_pos_only; ++args_copied) { const auto &arg_rec = func.args[args_copied]; - handle value; - if (arg_rec.value) { - value = arg_rec.value; - } - if (value) { - call.args.push_back(value); + call.args.push_back(arg_rec.value); call.args_convert.push_back(arg_rec.convert); } else { break; @@ -827,46 +1124,42 @@ class cpp_function : public function { } // 2. Check kwargs and, failing that, defaults that may help complete the list + small_vector used_kwargs( + kwnames_in ? static_cast(PyTuple_GET_SIZE(kwnames_in)) : 0, false); + size_t used_kwargs_count = 0; if (args_copied < num_args) { - bool copied_kwargs = false; - for (; args_copied < num_args; ++args_copied) { const auto &arg_rec = func.args[args_copied]; handle value; - if (kwargs_in && arg_rec.name) { - value = dict_getitemstring(kwargs.ptr(), arg_rec.name); + if (kwnames_in && arg_rec.name) { + ssize_t i = keyword_index(kwnames_in, arg_rec.name); + if (i >= 0) { + value = args_in_arr[n_args_in + static_cast(i)]; + used_kwargs.set(static_cast(i), true); + used_kwargs_count++; + } } - if (value) { - // Consume a kwargs value - if (!copied_kwargs) { - kwargs = reinterpret_steal(PyDict_Copy(kwargs.ptr())); - copied_kwargs = true; - } - if (PyDict_DelItemString(kwargs.ptr(), arg_rec.name) == -1) { - throw error_already_set(); - } - } else if (arg_rec.value) { + if (!value) { value = arg_rec.value; + if (!value) { + break; + } } if (!arg_rec.none && value.is_none()) { break; } - if (value) { - // If we're at the py::args index then first insert a stub for it to be - // replaced later - if (func.has_args && call.args.size() == func.nargs_pos) { - call.args.push_back(none()); - } - - call.args.push_back(value); - call.args_convert.push_back(arg_rec.convert); - } else { - break; + // If we're at the py::args index then first insert a stub for it to be + // replaced later + if (func.has_args && call.args.size() == func.nargs_pos) { + call.args.push_back(none()); } + + call.args.push_back(value); + call.args_convert.push_back(arg_rec.convert); } if (args_copied < num_args) { @@ -876,47 +1169,50 @@ class cpp_function : public function { } // 3. Check everything was consumed (unless we have a kwargs arg) - if (kwargs && !kwargs.empty() && !func.has_kwargs) { + if (!func.has_kwargs && used_kwargs_count < used_kwargs.size()) { continue; // Unconsumed kwargs, but no py::kwargs argument to accept them } // 4a. If we have a py::args argument, create a new tuple with leftovers if (func.has_args) { - tuple extra_args; - if (args_to_copy == 0) { - // We didn't copy out any position arguments from the args_in tuple, so we - // can reuse it directly without copying: - extra_args = reinterpret_borrow(args_in); - } else if (positional_args_copied >= n_args_in) { - extra_args = tuple(0); + if (positional_args_copied >= n_args_in) { + call.args_ref = tuple(0); } else { size_t args_size = n_args_in - positional_args_copied; - extra_args = tuple(args_size); + tuple extra_args(args_size); for (size_t i = 0; i < args_size; ++i) { - extra_args[i] = PyTuple_GET_ITEM(args_in, positional_args_copied + i); + extra_args[i] = args_in_arr[positional_args_copied + i]; } + call.args_ref = std::move(extra_args); } if (call.args.size() <= func.nargs_pos) { - call.args.push_back(extra_args); + call.args.push_back(call.args_ref); } else { - call.args[func.nargs_pos] = extra_args; + call.args[func.nargs_pos] = call.args_ref; } call.args_convert.push_back(false); - call.args_ref = std::move(extra_args); } // 4b. If we have a py::kwargs, pass on any remaining kwargs if (func.has_kwargs) { - if (!kwargs.ptr()) { - kwargs = dict(); // If we didn't get one, send an empty one + dict kwargs; + for (size_t i = 0; i < used_kwargs.size(); ++i) { + if (!used_kwargs[i]) { + // Cast values into handles before indexing into kwargs to ensure + // well-defined evaluation order (MSVC C4866). + handle arg_in_arr = args_in_arr[n_args_in + i], + kwname = PyTuple_GET_ITEM(kwnames_in, i); + kwargs[kwname] = arg_in_arr; + } } call.args.push_back(kwargs); call.args_convert.push_back(false); call.kwargs_ref = std::move(kwargs); } -// 5. Put everything in a vector. Not technically step 5, we've been building it -// in `call.args` all along. + // 5. Put everything in a vector. Not technically step 5, we've been building it + // in `call.args` all along. + #if defined(PYBIND11_DETAILED_ERROR_MESSAGES) if (call.args.size() != func.nargs || call.args_convert.size() != func.nargs) { pybind11_fail("Internal error: function call dispatcher inserted wrong number " @@ -924,13 +1220,14 @@ class cpp_function : public function { } #endif - std::vector second_pass_convert; + args_convert_vector second_pass_convert; if (overloaded) { // We're in the first no-convert pass, so swap out the conversion flags for a // set of all-false flags. If the call fails, we'll swap the flags back in for // the conversion-allowed call below. - second_pass_convert.resize(func.nargs, false); - call.args_convert.swap(second_pass_convert); + second_pass_convert = std::move(call.args_convert); + call.args_convert + = args_convert_vector(func.nargs, false); } // 6. Call the function. @@ -973,10 +1270,10 @@ class cpp_function : public function { } if (result.ptr() != PYBIND11_TRY_NEXT_OVERLOAD) { - // The error reporting logic below expects 'it' to be valid, as it would be - // if we'd encountered this failure in the first-pass loop. + // The error reporting logic below expects 'current_overload' to be valid, + // as it would be if we'd encountered this failure in the first-pass loop. if (!result) { - it = &call.func; + current_overload = &call.func; } break; } @@ -990,34 +1287,7 @@ class cpp_function : public function { throw; #endif } catch (...) { - /* When an exception is caught, give each registered exception - translator a chance to translate it to a Python exception. First - all module-local translators will be tried in reverse order of - registration. If none of the module-locale translators handle - the exception (or there are no module-locale translators) then - the global translators will be tried, also in reverse order of - registration. - - A translator may choose to do one of the following: - - - catch the exception and call PyErr_SetString or PyErr_SetObject - to set a standard (or custom) Python exception, or - - do nothing and let the exception fall through to the next translator, or - - delegate translation to the next translator by throwing a new type of exception. - */ - - auto &local_exception_translators - = get_local_internals().registered_exception_translators; - if (detail::apply_exception_translators(local_exception_translators)) { - return nullptr; - } - auto &exception_translators = get_internals().registered_exception_translators; - if (detail::apply_exception_translators(exception_translators)) { - return nullptr; - } - - PyErr_SetString(PyExc_SystemError, - "Exception escaped from default exception translator!"); + try_translate_exceptions(); return nullptr; } @@ -1073,40 +1343,37 @@ class cpp_function : public function { msg += '\n'; } msg += "\nInvoked with: "; - auto args_ = reinterpret_borrow(args_in); bool some_args = false; - for (size_t ti = overloads->is_constructor ? 1 : 0; ti < args_.size(); ++ti) { + for (size_t ti = overloads->is_constructor ? 1 : 0; ti < n_args_in; ++ti) { if (!some_args) { some_args = true; } else { msg += ", "; } try { - msg += pybind11::repr(args_[ti]); + msg += pybind11::repr(args_in_arr[ti]); } catch (const error_already_set &) { msg += ""; } } - if (kwargs_in) { - auto kwargs = reinterpret_borrow(kwargs_in); - if (!kwargs.empty()) { - if (some_args) { - msg += "; "; + if (kwnames_in && PyTuple_GET_SIZE(kwnames_in) > 0) { + if (some_args) { + msg += "; "; + } + msg += "kwargs: "; + bool first = true; + for (size_t i = 0; i < static_cast(PyTuple_GET_SIZE(kwnames_in)); ++i) { + if (first) { + first = false; + } else { + msg += ", "; } - msg += "kwargs: "; - bool first = true; - for (auto kwarg : kwargs) { - if (first) { - first = false; - } else { - msg += ", "; - } - msg += pybind11::str("{}=").format(kwarg.first); - try { - msg += pybind11::repr(kwarg.second); - } catch (const error_already_set &) { - msg += ""; - } + msg += reinterpret_borrow(PyTuple_GET_ITEM(kwnames_in, i)); + msg += '='; + try { + msg += pybind11::repr(args_in_arr[n_args_in + i]); + } catch (const error_already_set &) { + msg += ""; } } } @@ -1118,20 +1385,21 @@ class cpp_function : public function { raise_from(PyExc_TypeError, msg.c_str()); return nullptr; } - PyErr_SetString(PyExc_TypeError, msg.c_str()); + set_error(PyExc_TypeError, msg.c_str()); return nullptr; } if (!result) { std::string msg = "Unable to convert function return value to a " "Python type! The signature was\n\t"; - msg += it->signature; + assert(current_overload != nullptr); + msg += current_overload->signature; append_note_if_missing_header_is_suspected(msg); // Attach additional error info to the exception if supported if (PyErr_Occurred()) { raise_from(PyExc_TypeError, msg.c_str()); return nullptr; } - PyErr_SetString(PyExc_TypeError, msg.c_str()); + set_error(PyExc_TypeError, msg.c_str()); return nullptr; } if (overloads->is_constructor && !self_value_and_holder.holder_constructed()) { @@ -1140,7 +1408,221 @@ class cpp_function : public function { } return result.ptr(); } -}; + + static ssize_t keyword_index(PyObject *haystack, char const *needle) { + /* kwargs is usually very small (<= 5 entries). The arg strings are typically interned. + * CPython itself implements the search this way, first comparing all pointers ... which is + * cheap and will work if the strings are interned. If it fails, then it falls back to a + * second lexicographic check. This is wildly expensive for huge argument lists, but those + * are incredibly rare so we optimize for the vastly common case of just a couple of args. + */ + auto n = PyTuple_GET_SIZE(haystack); + auto s = reinterpret_steal(PyUnicode_InternFromString(needle)); + for (ssize_t i = 0; i < n; ++i) { + if (PyTuple_GET_ITEM(haystack, i) == s.ptr()) { + return i; + } + } + for (ssize_t i = 0; i < n; ++i) { + if (PyUnicode_Compare(PyTuple_GET_ITEM(haystack, i), s.ptr()) == 0) { + return i; + } + } + return -1; + } +}; + +PYBIND11_NAMESPACE_BEGIN(detail) + +PYBIND11_NAMESPACE_BEGIN(function_record_PyTypeObject_methods) + +// This implementation needs the definition of `class cpp_function`. +inline void tp_dealloc_impl(PyObject *self) { + // Save type before PyObject_Free invalidates self. + auto *type = Py_TYPE(self); + auto *py_func_rec = reinterpret_cast(self); + cpp_function::destruct(py_func_rec->cpp_func_rec); + py_func_rec->cpp_func_rec = nullptr; + // PyObject_New increments the heap type refcount and allocates via + // PyObject_Malloc; balance both here + PyObject_Free(self); + Py_DECREF(type); +} + +PYBIND11_NAMESPACE_END(function_record_PyTypeObject_methods) + +template <> +struct handle_type_name { + static constexpr auto name = const_name("collections.abc.Callable"); +}; + +PYBIND11_NAMESPACE_END(detail) + +// Use to activate Py_MOD_GIL_NOT_USED. +class mod_gil_not_used { +public: + mod_gil_not_used() : flag_(true) {} + PYBIND11_DEPRECATED("use py::mod_gil_not_used() or py::mod_gil_used() instead") + explicit mod_gil_not_used(bool flag) : flag_(flag) {} + bool flag() const { return flag_; } + + friend mod_gil_not_used mod_gil_used(); + +private: + bool flag_; +}; + +// Use to activate Py_MOD_GIL_USED, the current default. +inline mod_gil_not_used mod_gil_used() { + mod_gil_not_used tag; + tag.flag_ = false; + return tag; +} + +class multiple_interpreters { +public: + enum class level { + not_supported, /// Use to activate Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED + shared_gil, /// Use to activate Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED + per_interpreter_gil /// Use to activate Py_MOD_PER_INTERPRETER_GIL_SUPPORTED + }; + + static multiple_interpreters not_supported() { + return multiple_interpreters(level::not_supported); + } + static multiple_interpreters shared_gil() { return multiple_interpreters(level::shared_gil); } + static multiple_interpreters per_interpreter_gil() { + return multiple_interpreters(level::per_interpreter_gil); + } + + explicit constexpr multiple_interpreters(level l) : level_(l) {} + level value() const { return level_; } + +private: + level level_; +}; + +PYBIND11_NAMESPACE_BEGIN(detail) + +inline bool gil_not_used_option() { return false; } +template +bool gil_not_used_option(F &&, O &&...o); +template +inline bool gil_not_used_option(mod_gil_not_used f, O &&...o) { + return f.flag() || gil_not_used_option(o...); +} +template +inline bool gil_not_used_option(F &&, O &&...o) { + return gil_not_used_option(o...); +} + +#ifdef Py_mod_multiple_interpreters +inline void *multi_interp_slot() { return Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED; } +template +inline void *multi_interp_slot(multiple_interpreters mi, O &&...o) { + switch (mi.value()) { + case multiple_interpreters::level::per_interpreter_gil: + return Py_MOD_PER_INTERPRETER_GIL_SUPPORTED; + case multiple_interpreters::level::shared_gil: + return Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED; + case multiple_interpreters::level::not_supported: + return Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED; + } + // silence warnings with this unreachable line: + return multi_interp_slot(o...); +} +template +inline void *multi_interp_slot(F &&, O &&...o) { + return multi_interp_slot(o...); +} +#endif + +/* +Return a borrowed reference to the named module if it has been successfully initialized within this +interpreter before. nullptr if it has not been successfully initialized. +*/ +inline PyObject *get_cached_module(pybind11::str const &nameobj) { + dict state = detail::get_python_state_dict(); + if (!state.contains("__pybind11_module_cache")) { + return nullptr; + } + dict cache = state["__pybind11_module_cache"]; + if (!cache.contains(nameobj)) { + return nullptr; + } + return cache[nameobj].ptr(); +} + +/* +Add successfully initialized a module object to the internal cache. + +The module must have a __spec__ attribute with a name attribute. +*/ +inline void cache_completed_module(pybind11::object const &mod) { + dict state = detail::get_python_state_dict(); + if (!state.contains("__pybind11_module_cache")) { + state["__pybind11_module_cache"] = dict(); + } + state["__pybind11_module_cache"][mod.attr("__spec__").attr("name")] = mod; +} + +/* +A Py_mod_create slot function which will return the previously created module from the cache if one +exists, and otherwise will create a new module object. +*/ +inline PyObject *cached_create_module(PyObject *spec, PyModuleDef *) { + (void) &cache_completed_module; // silence unused-function warnings, it is used in a macro + + auto nameobj = getattr(reinterpret_borrow(spec), "name", none()); + if (nameobj.is_none()) { + set_error(PyExc_ImportError, "module spec is missing a name"); + return nullptr; + } + + auto *mod = get_cached_module(nameobj); + if (mod) { + Py_INCREF(mod); + } else { + mod = PyModule_NewObject(nameobj.ptr()); + } + return mod; +} + +/// Must be a POD type, and must hold enough entries for all of the possible slots PLUS ONE for +/// the sentinel (0) end slot. +using slots_array = std::array; + +/// Initialize an array of slots based on the supplied exec slot and options. +template +inline slots_array init_slots(int (*exec_fn)(PyObject *), Options &&...options) noexcept { + /* NOTE: slots_array MUST be large enough to hold all possible options. If you add an option + here, you MUST also increase the size of slots_array in the type alias above! */ + slots_array mod_def_slots; + size_t next_slot = 0; + + mod_def_slots[next_slot++] = {Py_mod_create, reinterpret_cast(&cached_create_module)}; + + if (exec_fn != nullptr) { + mod_def_slots[next_slot++] = {Py_mod_exec, reinterpret_cast(exec_fn)}; + } + +#ifdef Py_mod_multiple_interpreters + mod_def_slots[next_slot++] = {Py_mod_multiple_interpreters, multi_interp_slot(options...)}; +#endif + + if (gil_not_used_option(options...)) { +#if defined(Py_mod_gil) && defined(Py_GIL_DISABLED) + mod_def_slots[next_slot++] = {Py_mod_gil, Py_MOD_GIL_NOT_USED}; +#endif + } + + // slots must have a zero end sentinel + mod_def_slots[next_slot++] = {0, nullptr}; + + return mod_def_slots; +} + +PYBIND11_NAMESPACE_END(detail) /// Wrapper for Python extension modules class module_ : public object { @@ -1196,6 +1678,24 @@ class module_ : public object { if (doc && options::show_user_defined_docstrings()) { result.attr("__doc__") = pybind11::str(doc); } + +#if defined(GRAALVM_PYTHON) && (!defined(GRAALPY_VERSION_NUM) || GRAALPY_VERSION_NUM < 0x190000) + // GraalPy doesn't support PyModule_GetFilenameObject, + // so getting by attribute (see PR #5584) + handle this_module = m_ptr; + if (object this_file = getattr(this_module, "__file__", none())) { + result.attr("__file__") = this_file; + } +#else + handle this_file = PyModule_GetFilenameObject(m_ptr); + if (this_file) { + result.attr("__file__") = this_file; + } else if (PyErr_ExceptionMatches(PyExc_SystemError) != 0) { + PyErr_Clear(); + } else { + throw error_already_set(); + } +#endif attr(name) = result; return result; } @@ -1235,26 +1735,28 @@ class module_ : public object { PyModule_AddObject(ptr(), name, obj.inc_ref().ptr() /* steals a reference */); } - using module_def = PyModuleDef; // TODO: Can this be removed (it was needed only for Python 2)? + // DEPRECATED (since PR #5688): Use PyModuleDef directly instead. + using module_def = PyModuleDef; /** \rst Create a new top-level module that can be used as the main module of a C extension. - ``def`` should point to a statically allocated module_def. + ``def`` should point to a statically allocated PyModuleDef. \endrst */ - static module_ create_extension_module(const char *name, const char *doc, module_def *def) { - // module_def is PyModuleDef + static module_ create_extension_module(const char *name, + const char *doc, + PyModuleDef *def, + mod_gil_not_used gil_not_used = mod_gil_used()) { // Placement new (not an allocation). - def = new (def) - PyModuleDef{/* m_base */ PyModuleDef_HEAD_INIT, - /* m_name */ name, - /* m_doc */ options::show_user_defined_docstrings() ? doc : nullptr, - /* m_size */ -1, - /* m_methods */ nullptr, - /* m_slots */ nullptr, - /* m_traverse */ nullptr, - /* m_clear */ nullptr, - /* m_free */ nullptr}; + new (def) PyModuleDef{/* m_base */ PyModuleDef_HEAD_INIT, + /* m_name */ name, + /* m_doc */ options::show_user_defined_docstrings() ? doc : nullptr, + /* m_size */ -1, + /* m_methods */ nullptr, + /* m_slots */ nullptr, + /* m_traverse */ nullptr, + /* m_clear */ nullptr, + /* m_free */ nullptr}; auto *m = PyModule_Create(def); if (m == nullptr) { if (PyErr_Occurred()) { @@ -1262,6 +1764,11 @@ class module_ : public object { } pybind11_fail("Internal error in module_::create_extension_module()"); } + if (gil_not_used.flag()) { +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif + } // TODO: Should be reinterpret_steal for Python 3, but Python also steals it again when // returned from PyInit_... // For Python 2, reinterpret_borrow was correct. @@ -1269,6 +1776,15 @@ class module_ : public object { } }; +PYBIND11_NAMESPACE_BEGIN(detail) + +template <> +struct handle_type_name { + static constexpr auto name = const_name("types.ModuleType"); +}; + +PYBIND11_NAMESPACE_END(detail) + // When inside a namespace (or anywhere as long as it's not the first item on a line), // C++20 allows "module" to be used. This is provided for backward compatibility, and for // simplicity, if someone wants to use py::module for example, that is perfectly safe. @@ -1278,15 +1794,14 @@ using module = module_; /// Return a dictionary representing the global variables in the current execution frame, /// or ``__main__.__dict__`` if there is no frame (usually when the interpreter is embedded). inline dict globals() { +#if PY_VERSION_HEX >= 0x030d0000 + PyObject *p = PyEval_GetFrameGlobals(); + return p ? reinterpret_steal(p) + : reinterpret_borrow(module_::import("__main__").attr("__dict__").ptr()); +#else PyObject *p = PyEval_GetGlobals(); return reinterpret_borrow(p ? p : module_::import("__main__").attr("__dict__").ptr()); -} - -template ()>> -PYBIND11_DEPRECATED("make_simple_namespace should be replaced with " - "py::module_::import(\"types\").attr(\"SimpleNamespace\") ") -object make_simple_namespace(Args &&...args_) { - return module_::import("types").attr("SimpleNamespace")(std::forward(args_)...); +#endif } PYBIND11_NAMESPACE_BEGIN(detail) @@ -1312,7 +1827,7 @@ class generic_type : public object { /* Register supplemental type information in C++ dict */ auto *tinfo = new detail::type_info(); - tinfo->type = (PyTypeObject *) m_ptr; + tinfo->type = reinterpret_cast(m_ptr); tinfo->cpptype = rec.type; tinfo->type_size = rec.type_size; tinfo->type_align = rec.type_align; @@ -1320,26 +1835,43 @@ class generic_type : public object { tinfo->holder_size_in_ptrs = size_in_ptrs(rec.holder_size); tinfo->init_instance = rec.init_instance; tinfo->dealloc = rec.dealloc; + tinfo->get_trampoline_self_life_support = rec.get_trampoline_self_life_support; tinfo->simple_type = true; tinfo->simple_ancestors = true; - tinfo->default_holder = rec.default_holder; tinfo->module_local = rec.module_local; + tinfo->holder_enum_v = rec.holder_enum_v; + + with_internals([&](internals &internals) { + auto tindex = std::type_index(*rec.type); + tinfo->direct_conversions = &internals.direct_conversions[tindex]; + auto &local_internals = get_local_internals(); + if (rec.module_local) { + local_internals.registered_types_cpp[rec.type] = tinfo; + } else { + internals.registered_types_cpp[tindex] = tinfo; +#if PYBIND11_INTERNALS_VERSION >= 12 + internals.registered_types_cpp_fast[rec.type] = tinfo; +#endif + } - auto &internals = get_internals(); - auto tindex = std::type_index(*rec.type); - tinfo->direct_conversions = &internals.direct_conversions[tindex]; - if (rec.module_local) { - get_local_internals().registered_types_cpp[tindex] = tinfo; - } else { - internals.registered_types_cpp[tindex] = tinfo; - } - internals.registered_types_py[(PyTypeObject *) m_ptr] = {tinfo}; + PYBIND11_WARNING_PUSH +#if defined(__GNUC__) && __GNUC__ == 12 + // When using GCC 12 these warnings are disabled as they trigger + // false positive warnings. Discussed here: + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=115824. + PYBIND11_WARNING_DISABLE_GCC("-Warray-bounds") + PYBIND11_WARNING_DISABLE_GCC("-Wstringop-overread") +#endif + internals.registered_types_py[reinterpret_cast(m_ptr)] = {tinfo}; + PYBIND11_WARNING_POP + }); if (rec.bases.size() > 1 || rec.multiple_inheritance) { mark_parents_nonsimple(tinfo->type); tinfo->simple_ancestors = false; } else if (rec.bases.size() == 1) { - auto *parent_tinfo = get_type_info((PyTypeObject *) rec.bases[0].ptr()); + auto *parent_tinfo + = get_type_info(reinterpret_cast(rec.bases[0].ptr())); assert(parent_tinfo != nullptr); bool parent_simple_ancestors = parent_tinfo->simple_ancestors; tinfo->simple_ancestors = parent_simple_ancestors; @@ -1358,17 +1890,17 @@ class generic_type : public object { void mark_parents_nonsimple(PyTypeObject *value) { auto t = reinterpret_borrow(value->tp_bases); for (handle h : t) { - auto *tinfo2 = get_type_info((PyTypeObject *) h.ptr()); + auto *tinfo2 = get_type_info(reinterpret_cast(h.ptr())); if (tinfo2) { tinfo2->simple_type = false; } - mark_parents_nonsimple((PyTypeObject *) h.ptr()); + mark_parents_nonsimple(reinterpret_cast(h.ptr())); } } void install_buffer_funcs(buffer_info *(*get_buffer)(PyObject *, void *), void *get_buffer_data) { - auto *type = (PyHeapTypeObject *) m_ptr; + auto *type = reinterpret_cast(m_ptr); auto *tinfo = detail::get_type_info(&type->ht_type); if (!type->ht_type.tp_as_buffer) { @@ -1390,8 +1922,8 @@ class generic_type : public object { const auto is_static = (rec_func != nullptr) && !(rec_func->is_method && rec_func->scope); const auto has_doc = (rec_func != nullptr) && (rec_func->doc != nullptr) && pybind11::options::show_user_defined_docstrings(); - auto property = handle( - (PyObject *) (is_static ? get_internals().static_property_type : &PyProperty_Type)); + auto property = handle(reinterpret_cast( + is_static ? get_internals().static_property_type : &PyProperty_Type)); attr(name) = property(fget.ptr() ? fget : none(), fset.ptr() ? fset : none(), /*deleter*/ none(), @@ -1436,7 +1968,7 @@ void call_operator_delete(T *p, size_t s, size_t) { inline void call_operator_delete(void *p, size_t s, size_t a) { (void) s; (void) a; -#if defined(__cpp_aligned_new) && (!defined(_MSC_VER) || _MSC_VER >= 1912) +#if defined(__cpp_aligned_new) if (a > __STDCPP_DEFAULT_NEW_ALIGNMENT__) { # ifdef __cpp_sized_deallocation ::operator delete(p, s, std::align_val_t(a)); @@ -1460,30 +1992,323 @@ inline void add_class_method(object &cls, const char *name_, const cpp_function } } +/// Type trait to rebind a member function pointer's class to `Derived`, preserving all +/// cv/ref/noexcept qualifiers. The primary template has no `type` member, providing SFINAE +/// failure for unsupported member function pointer types. `source_class` holds the original +/// class for use in `is_accessible_base_of` checks. +template +struct rebind_member_ptr {}; + +// Define one specialization per supported qualifier combination via a local macro. +// The qualifiers argument appears in type position, not expression position, so +// parenthesizing it would produce invalid C++. +// The no-qualifier specialization is written out explicitly to avoid invoking the macro with an +// empty argument, which triggers MSVC warning C4003. +template +struct rebind_member_ptr { + using type = Return (Derived::*)(Args...); + using source_class = Class; +}; +// NOLINTBEGIN(bugprone-macro-parentheses) +#define PYBIND11_REBIND_MEMBER_PTR(qualifiers) \ + template \ + struct rebind_member_ptr { \ + using type = Return (Derived::*)(Args...) qualifiers; \ + using source_class = Class; \ + } +PYBIND11_REBIND_MEMBER_PTR(const); +PYBIND11_REBIND_MEMBER_PTR(&); +PYBIND11_REBIND_MEMBER_PTR(const &); +PYBIND11_REBIND_MEMBER_PTR(&&); +PYBIND11_REBIND_MEMBER_PTR(const &&); +#ifdef __cpp_noexcept_function_type +PYBIND11_REBIND_MEMBER_PTR(noexcept); +PYBIND11_REBIND_MEMBER_PTR(const noexcept); +PYBIND11_REBIND_MEMBER_PTR(& noexcept); +PYBIND11_REBIND_MEMBER_PTR(const & noexcept); +PYBIND11_REBIND_MEMBER_PTR(&& noexcept); +PYBIND11_REBIND_MEMBER_PTR(const && noexcept); +#endif +#undef PYBIND11_REBIND_MEMBER_PTR +// NOLINTEND(bugprone-macro-parentheses) + +/// Shared implementation body for all method_adaptor member-function-pointer overloads. +/// Asserts Base is accessible from Derived, then casts the member pointer. +template , + typename Adapted = typename Traits::type> +constexpr PYBIND11_ALWAYS_INLINE Adapted adapt_member_ptr(T pmf) { + static_assert( + detail::is_accessible_base_of::value, + "Cannot bind an inaccessible base class method; use a lambda definition instead"); + return pmf; +} + PYBIND11_NAMESPACE_END(detail) /// Given a pointer to a member function, cast it to its `Derived` version. -/// Forward everything else unchanged. -template -auto method_adaptor(F &&f) -> decltype(std::forward(f)) { +/// For all other callables (lambdas, function pointers, etc.), forward unchanged. +/// +/// Two overloads cover all cases without explicit per-qualifier instantiations: +/// +/// (1) Generic fallback — disabled for member function pointers so that (2) wins +/// without any partial-ordering ambiguity. +/// (2) MFP overload — SFINAE on rebind_member_ptr::type, which exists for every +/// supported qualifier combination (const, &, &&, noexcept, ...). A single +/// template therefore covers all combinations that rebind_member_ptr handles. +template < + typename /*Derived*/, + typename F, + detail::enable_if_t>::value, + int> = 0> +constexpr auto method_adaptor(F &&f) -> decltype(std::forward(f)) { return std::forward(f); } -template -auto method_adaptor(Return (Class::*pmf)(Args...)) -> Return (Derived::*)(Args...) { - static_assert( - detail::is_accessible_base_of::value, - "Cannot bind an inaccessible base class method; use a lambda definition instead"); - return pmf; +template ::type> +constexpr Adapted method_adaptor(T pmf) { + // Expected to be redundant (SFINAE on rebind_member_ptr) but cheap and makes the intent + // explicit. + static_assert(std::is_member_function_pointer::value, + "method_adaptor: T must be a member function pointer"); + return detail::adapt_member_ptr(pmf); } -template -auto method_adaptor(Return (Class::*pmf)(Args...) const) -> Return (Derived::*)(Args...) const { - static_assert( - detail::is_accessible_base_of::value, - "Cannot bind an inaccessible base class method; use a lambda definition instead"); - return pmf; -} +PYBIND11_NAMESPACE_BEGIN(detail) + +// Helper for the property_cpp_function static member functions below. +// The only purpose of these functions is to support .def_readonly & .def_readwrite. +// In this context, the PM template parameter is certain to be a Pointer to a Member. +// The main purpose of must_be_member_function_pointer is to make this obvious, and to guard +// against accidents. As a side-effect, it also explains why the syntactical overhead for +// perfect forwarding is not needed. +template +using must_be_member_function_pointer = enable_if_t::value, int>; + +// Note that property_cpp_function is intentionally in the main pybind11 namespace, +// because user-defined specializations could be useful. + +// Classic (non-smart_holder) implementations for .def_readonly and .def_readwrite +// getter and setter functions. +// WARNING: This classic implementation can lead to dangling pointers for raw pointer members. +// See test_ptr() in tests/test_class_sh_property.py +// However, this implementation works as-is (and safely) for smart_holder std::shared_ptr members. +template +struct property_cpp_function_classic { + template = 0> + static cpp_function readonly(PM pm, const handle &hdl) { + return cpp_function([pm](const T &c) -> const D & { return c.*pm; }, is_method(hdl)); + } + + template = 0> + static cpp_function read(PM pm, const handle &hdl) { + return readonly(pm, hdl); + } + + template = 0> + static cpp_function write(PM pm, const handle &hdl) { + return cpp_function([pm](T &c, const D &value) { c.*pm = value; }, is_method(hdl)); + } +}; + +PYBIND11_NAMESPACE_END(detail) + +template +struct property_cpp_function : detail::property_cpp_function_classic {}; + +PYBIND11_NAMESPACE_BEGIN(detail) + +template +struct both_t_and_d_use_type_caster_base : std::false_type {}; + +// `T` is assumed to be equivalent to `intrinsic_t`. +// `D` is may or may not be equivalent to `intrinsic_t`. +template +struct both_t_and_d_use_type_caster_base< + T, + D, + enable_if_t, type_caster>, + std::is_base_of>, make_caster>>::value>> + : std::true_type {}; + +// Specialization for raw pointer members, using smart_holder if that is the class_ holder, +// or falling back to the classic implementation if not. +// WARNING: Like the classic implementation, this implementation can lead to dangling pointers. +// See test_ptr() in tests/test_class_sh_property.py +// However, the read functions return a shared_ptr to the member, emulating the PyCLIF approach: +// https://github.com/google/clif/blob/c371a6d4b28d25d53a16e6d2a6d97305fb1be25a/clif/python/instance.h#L233 +// This prevents disowning of the Python object owning the raw pointer member. +template +struct property_cpp_function_sh_raw_ptr_member { + using drp = typename std::remove_pointer::type; + + template = 0> + static cpp_function readonly(PM pm, const handle &hdl) { + type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true); + if (tinfo->holder_enum_v == holder_enum_t::smart_holder) { + return cpp_function( + [pm](handle c_hdl) -> std::shared_ptr { + std::shared_ptr c_sp + = type_caster>::shared_ptr_with_responsible_parent( + c_hdl); + D ptr = (*c_sp).*pm; + return std::shared_ptr(c_sp, ptr); + }, + is_method(hdl)); + } + return property_cpp_function_classic::readonly(pm, hdl); + } + + template = 0> + static cpp_function read(PM pm, const handle &hdl) { + return readonly(pm, hdl); + } + + template = 0> + static cpp_function write(PM pm, const handle &hdl) { + type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true); + if (tinfo->holder_enum_v == holder_enum_t::smart_holder) { + return cpp_function([pm](T &c, D value) { c.*pm = std::forward(std::move(value)); }, + is_method(hdl)); + } + return property_cpp_function_classic::write(pm, hdl); + } +}; + +// Specialization for members held by-value, using smart_holder if that is the class_ holder, +// or falling back to the classic implementation if not. +// The read functions return a shared_ptr to the member, emulating the PyCLIF approach: +// https://github.com/google/clif/blob/c371a6d4b28d25d53a16e6d2a6d97305fb1be25a/clif/python/instance.h#L233 +// This prevents disowning of the Python object owning the member. +template +struct property_cpp_function_sh_member_held_by_value { + template = 0> + static cpp_function readonly(PM pm, const handle &hdl) { + type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true); + if (tinfo->holder_enum_v == holder_enum_t::smart_holder) { + return cpp_function( + [pm](handle c_hdl) -> std::shared_ptr::type> { + std::shared_ptr c_sp + = type_caster>::shared_ptr_with_responsible_parent( + c_hdl); + return std::shared_ptr::type>(c_sp, + &(c_sp.get()->*pm)); + }, + is_method(hdl)); + } + return property_cpp_function_classic::readonly(pm, hdl); + } + + template = 0> + static cpp_function read(PM pm, const handle &hdl) { + type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true); + if (tinfo->holder_enum_v == holder_enum_t::smart_holder) { + return cpp_function( + [pm](handle c_hdl) -> std::shared_ptr { + std::shared_ptr c_sp + = type_caster>::shared_ptr_with_responsible_parent( + c_hdl); + return std::shared_ptr(c_sp, &(c_sp.get()->*pm)); + }, + is_method(hdl)); + } + return property_cpp_function_classic::read(pm, hdl); + } + + template = 0> + static cpp_function write(PM pm, const handle &hdl) { + type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true); + if (tinfo->holder_enum_v == holder_enum_t::smart_holder) { + return cpp_function([pm](T &c, const D &value) { c.*pm = value; }, is_method(hdl)); + } + return property_cpp_function_classic::write(pm, hdl); + } +}; + +// Specialization for std::unique_ptr members, using smart_holder if that is the class_ holder, +// or falling back to the classic implementation if not. +// read disowns the member unique_ptr. +// write disowns the passed Python object. +// readonly is disabled (static_assert) because there is no safe & intuitive way to make the member +// accessible as a Python object without disowning the member unique_ptr. A .def_readonly disowning +// the unique_ptr member is deemed highly prone to misunderstandings. +template +struct property_cpp_function_sh_unique_ptr_member { + template = 0> + static cpp_function readonly(PM, const handle &) { + static_assert(!is_instantiation::value, + "def_readonly cannot be used for std::unique_ptr members."); + return cpp_function{}; // Unreachable. + } + + template = 0> + static cpp_function read(PM pm, const handle &hdl) { + type_info *tinfo = get_type_info(typeid(T), /*throw_if_missing=*/true); + if (tinfo->holder_enum_v == holder_enum_t::smart_holder) { + return cpp_function( + [pm](handle c_hdl) -> D { + std::shared_ptr c_sp + = type_caster>::shared_ptr_with_responsible_parent( + c_hdl); + return D{std::move(c_sp.get()->*pm)}; + }, + is_method(hdl)); + } + return property_cpp_function_classic::read(pm, hdl); + } + + template = 0> + static cpp_function write(PM pm, const handle &hdl) { + return cpp_function([pm](T &c, D &&value) { c.*pm = std::move(value); }, is_method(hdl)); + } +}; + +PYBIND11_NAMESPACE_END(detail) + +template +struct property_cpp_function< + T, + D, + detail::enable_if_t, + detail::both_t_and_d_use_type_caster_base>::value>> + : detail::property_cpp_function_sh_raw_ptr_member {}; + +template +struct property_cpp_function, + std::is_array, + detail::is_instantiation, + detail::is_instantiation>, + detail::both_t_and_d_use_type_caster_base>::value>> + : detail::property_cpp_function_sh_member_held_by_value {}; + +template +struct property_cpp_function< + T, + D, + detail::enable_if_t, + detail::both_t_and_d_use_type_caster_base>::value>> + : detail::property_cpp_function_sh_unique_ptr_member {}; + +#ifdef PYBIND11_RUN_TESTING_WITH_SMART_HOLDER_AS_DEFAULT_BUT_NEVER_USE_IN_PRODUCTION_PLEASE +// NOTE: THIS IS MEANT FOR STRESS-TESTING OR TRIAGING ONLY! +// Running the pybind11 unit tests with smart_holder as the default holder is to ensure +// that `py::smart_holder` / `py::classh` is backward-compatible with all pre-existing +// functionality. +// Be careful not to link translation units compiled with different default holders, because +// this will cause ODR violations (https://en.wikipedia.org/wiki/One_Definition_Rule). +template +using default_holder_type = smart_holder; +#else +template +using default_holder_type = std::unique_ptr; +#endif template class class_ : public detail::generic_type { @@ -1501,13 +2326,27 @@ class class_ : public detail::generic_type { using type = type_; using type_alias = detail::exactly_one_t; constexpr static bool has_alias = !std::is_void::value; - using holder_type = detail::exactly_one_t, options...>; + using holder_type = detail::exactly_one_t, options...>; static_assert(detail::all_of...>::value, "Unknown/invalid class_ template parameters provided"); static_assert(!has_alias || std::is_polymorphic::value, - "Cannot use an alias class with a non-polymorphic type"); + "Cannot use an alias class (aka trampoline) with a non-polymorphic type"); + +#ifndef PYBIND11_RUN_TESTING_WITH_SMART_HOLDER_AS_DEFAULT_BUT_NEVER_USE_IN_PRODUCTION_PLEASE + static_assert(!has_alias || !detail::is_smart_holder::value + || std::is_base_of::value, + "Alias class (aka trampoline) must inherit from" + " pybind11::trampoline_self_life_support if used in combination with" + " pybind11::smart_holder"); +#endif + static_assert(!has_alias || detail::is_smart_holder::value + || !std::is_base_of::value, + "pybind11::trampoline_self_life_support is a smart_holder feature, therefore" + " an alias class (aka trampoline) should inherit from" + " pybind11::trampoline_self_life_support only if used in combination with" + " pybind11::smart_holder"); PYBIND11_OBJECT(class_, generic_type, PyType_Check) @@ -1532,8 +2371,16 @@ class class_ : public detail::generic_type { record.type_align = alignof(conditional_t &); record.holder_size = sizeof(holder_type); record.init_instance = init_instance; - record.dealloc = dealloc; - record.default_holder = detail::is_instantiation::value; + + if (detail::is_instantiation::value) { + record.holder_enum_v = detail::holder_enum_t::std_unique_ptr; + } else if (detail::is_instantiation::value) { + record.holder_enum_v = detail::holder_enum_t::std_shared_ptr; + } else if (std::is_same::value) { + record.holder_enum_v = detail::holder_enum_t::smart_holder; + } else { + record.holder_enum_v = detail::holder_enum_t::custom_holder; + } set_operator_new(&record); @@ -1543,14 +2390,41 @@ class class_ : public detail::generic_type { /* Process optional arguments, if any */ process_attributes::init(extra..., &record); + if (record.release_gil_before_calling_cpp_dtor) { + record.dealloc = dealloc_release_gil_before_calling_cpp_dtor; + } else { + record.dealloc = dealloc_without_manipulating_gil; + } + + if (std::is_base_of::value) { + // Store a cross-DSO-safe getter. + // This lambda is defined in the same DSO that instantiates + // class_, but it can be called safely from any other DSO. + record.get_trampoline_self_life_support = [](void *type_ptr) { + return dynamic_raw_ptr_cast_if_possible( + static_cast(type_ptr)); + }; + } + generic_type::initialize(record); if (has_alias) { - auto &instances = record.module_local ? get_local_internals().registered_types_cpp - : get_internals().registered_types_cpp; - instances[std::type_index(typeid(type_alias))] - = instances[std::type_index(typeid(type))]; + with_internals([&](internals &internals) { + auto &local_internals = get_local_internals(); + if (record.module_local) { + local_internals.registered_types_cpp[&typeid(type_alias)] + = local_internals.registered_types_cpp[&typeid(type)]; + } else { + type_info *const val + = internals.registered_types_cpp[std::type_index(typeid(type))]; + internals.registered_types_cpp[std::type_index(typeid(type_alias))] = val; +#if PYBIND11_INTERNALS_VERSION >= 12 + internals.registered_types_cpp_fast[&typeid(type_alias)] = val; +#endif + } + }); } + def("_pybind11_conduit_v1_", cpp_conduit_method); } template ::value, int> = 0> @@ -1558,13 +2432,20 @@ class class_ : public detail::generic_type { rec.add_base(typeid(Base), [](void *src) -> void * { return static_cast(reinterpret_cast(src)); }); + // Virtual inheritance means the base subobject is at a dynamic offset, + // so the reinterpret_cast shortcut in load_impl Case 2a is invalid. + // Force the MI path (implicit_casts) for correct pointer adjustment. + // Detection: static_cast(Base*) is ill-formed for virtual bases. + if PYBIND11_MAYBE_CONSTEXPR (!detail::is_static_downcastable::value) { + rec.multiple_inheritance = true; + } } template ::value, int> = 0> static void add_base(detail::type_record &) {} template - class_ &def(const char *name_, Func &&f, const Extra &...extra) { + PYBIND11_ALWAYS_INLINE class_ &def(const char *name_, Func &&f, const Extra &...extra) { cpp_function cf(method_adaptor(std::forward(f)), name(name_), is_method(*this), @@ -1659,13 +2540,49 @@ class class_ : public detail::generic_type { return def_buffer([func](const type &obj) { return (obj.*func)(); }); } + // Intentionally no &&/const&& overloads: buffer protocol callbacks are invoked on an + // existing Python object and should not move-from self. + template + class_ &def_buffer(Return (Class::*func)(Args...) &) { + return def_buffer([func](type &obj) { return (obj.*func)(); }); + } + + template + class_ &def_buffer(Return (Class::*func)(Args...) const &) { + return def_buffer([func](const type &obj) { return (obj.*func)(); }); + } + +#ifdef __cpp_noexcept_function_type + template + class_ &def_buffer(Return (Class::*func)(Args...) noexcept) { + return def_buffer([func](type &obj) { return (obj.*func)(); }); + } + + template + class_ &def_buffer(Return (Class::*func)(Args...) const noexcept) { + return def_buffer([func](const type &obj) { return (obj.*func)(); }); + } + + template + class_ &def_buffer(Return (Class::*func)(Args...) & noexcept) { + return def_buffer([func](type &obj) { return (obj.*func)(); }); + } + + template + class_ &def_buffer(Return (Class::*func)(Args...) const & noexcept) { + return def_buffer([func](const type &obj) { return (obj.*func)(); }); + } +#endif + template class_ &def_readwrite(const char *name, D C::*pm, const Extra &...extra) { static_assert(std::is_same::value || std::is_base_of::value, "def_readwrite() requires a class member (or base class member)"); - cpp_function fget([pm](const type &c) -> const D & { return c.*pm; }, is_method(*this)), - fset([pm](type &c, const D &value) { c.*pm = value; }, is_method(*this)); - def_property(name, fget, fset, return_value_policy::reference_internal, extra...); + def_property(name, + property_cpp_function::read(pm, *this), + property_cpp_function::write(pm, *this), + return_value_policy::reference_internal, + extra...); return *this; } @@ -1673,8 +2590,10 @@ class class_ : public detail::generic_type { class_ &def_readonly(const char *name, const D C::*pm, const Extra &...extra) { static_assert(std::is_same::value || std::is_base_of::value, "def_readonly() requires a class member (or base class member)"); - cpp_function fget([pm](const type &c) -> const D & { return c.*pm; }, is_method(*this)); - def_property_readonly(name, fget, return_value_policy::reference_internal, extra...); + def_property_readonly(name, + property_cpp_function::readonly(pm, *this), + return_value_policy::reference_internal, + extra...); return *this; } @@ -1729,7 +2648,8 @@ class class_ : public detail::generic_type { template class_ & def_property(const char *name, const Getter &fget, const Setter &fset, const Extra &...extra) { - return def_property(name, fget, cpp_function(method_adaptor(fset)), extra...); + return def_property( + name, fget, cpp_function(method_adaptor(fset), is_setter()), extra...); } template class_ &def_property(const char *name, @@ -1770,24 +2690,52 @@ class class_ : public detail::generic_type { const Extra &...extra) { static_assert(0 == detail::constexpr_sum(std::is_base_of::value...), "Argument annotations are not allowed for properties"); + static_assert(0 == detail::constexpr_sum(detail::is_call_guard::value...), + "def_property family does not currently support call_guard. Use a " + "py::cpp_function instead."); + static_assert(0 == detail::constexpr_sum(detail::is_keep_alive::value...), + "def_property family does not currently support keep_alive. Use a " + "py::cpp_function instead."); auto rec_fget = get_function_record(fget), rec_fset = get_function_record(fset); auto *rec_active = rec_fget; if (rec_fget) { char *doc_prev = rec_fget->doc; /* 'extra' field may include a property-specific documentation string */ + auto args_before = rec_fget->args.size(); detail::process_attributes::init(extra..., rec_fget); if (rec_fget->doc && rec_fget->doc != doc_prev) { std::free(doc_prev); rec_fget->doc = PYBIND11_COMPAT_STRDUP(rec_fget->doc); } + // Args added by process_attributes (e.g. "self" via is_method + pos_only/kw_only) + // need their strings strdup'd: initialize_generic's strdup loop already ran during + // cpp_function construction, so it won't process these late additions. Without this, + // destruct() would call free() on string literals. See gh-5976. + for (auto i = args_before; i < rec_fget->args.size(); ++i) { + if (rec_fget->args[i].name) { + rec_fget->args[i].name = PYBIND11_COMPAT_STRDUP(rec_fget->args[i].name); + } + if (rec_fget->args[i].descr) { + rec_fget->args[i].descr = PYBIND11_COMPAT_STRDUP(rec_fget->args[i].descr); + } + } } if (rec_fset) { char *doc_prev = rec_fset->doc; + auto args_before = rec_fset->args.size(); detail::process_attributes::init(extra..., rec_fset); if (rec_fset->doc && rec_fset->doc != doc_prev) { std::free(doc_prev); rec_fset->doc = PYBIND11_COMPAT_STRDUP(rec_fset->doc); } + for (auto i = args_before; i < rec_fset->args.size(); ++i) { + if (rec_fset->args[i].name) { + rec_fset->args[i].name = PYBIND11_COMPAT_STRDUP(rec_fset->args[i].name); + } + if (rec_fset->args[i].descr) { + rec_fset->args[i].descr = PYBIND11_COMPAT_STRDUP(rec_fset->args[i].descr); + } + } if (!rec_active) { rec_active = rec_fset; } @@ -1820,8 +2768,7 @@ class class_ : public detail::generic_type { static void init_holder_from_existing(const detail::value_and_holder &v_h, const holder_type *holder_ptr, std::true_type /*is_copy_constructible*/) { - new (std::addressof(v_h.holder())) - holder_type(*reinterpret_cast(holder_ptr)); + new (std::addressof(v_h.holder())) holder_type(*holder_ptr); } static void init_holder_from_existing(const detail::value_and_holder &v_h, @@ -1850,6 +2797,8 @@ class class_ : public detail::generic_type { /// instance. Should be called as soon as the `type` value_ptr is set for an instance. Takes /// an optional pointer to an existing holder to use; if not specified and the instance is /// `.owned`, a new holder will be constructed to manage the value pointer. + template ::value, int> = 0> static void init_instance(detail::instance *inst, const void *holder_ptr) { auto v_h = inst->get_value_and_holder(detail::get_type_info(typeid(type))); if (!v_h.instance_registered()) { @@ -1859,15 +2808,73 @@ class class_ : public detail::generic_type { init_holder(inst, v_h, (const holder_type *) holder_ptr, v_h.value_ptr()); } - /// Deallocates an instance; via holder, if constructed; otherwise via operator delete. - static void dealloc(detail::value_and_holder &v_h) { - // We could be deallocating because we are cleaning up after a Python exception. - // If so, the Python error indicator will be set. We need to clear that before - // running the destructor, in case the destructor code calls more Python. - // If we don't, the Python API will exit with an exception, and pybind11 will - // throw error_already_set from the C++ destructor which is forbidden and triggers - // std::terminate(). - error_scope scope; + template + static bool try_initialization_using_shared_from_this(holder_type *, WrappedType *, ...) { + return false; + } + + // Adopting existing approach used by type_caster_base, although it leads to somewhat fuzzy + // ownership semantics: if we detected via shared_from_this that a shared_ptr exists already, + // it is reused, irrespective of the return_value_policy in effect. + // "SomeBaseOfWrappedType" is needed because std::enable_shared_from_this is not necessarily a + // direct base of WrappedType. + template + static bool try_initialization_using_shared_from_this( + holder_type *uninitialized_location, + WrappedType *value_ptr_w_t, + const std::enable_shared_from_this *) { + auto shd_ptr = std::dynamic_pointer_cast( + detail::try_get_shared_from_this(value_ptr_w_t)); + if (!shd_ptr) { + return false; + } + // Note: inst->owned ignored. + new (uninitialized_location) holder_type(holder_type::from_shared_ptr(shd_ptr)); + return true; + } + + template ::value, int> = 0> + static void init_instance(detail::instance *inst, const void *holder_const_void_ptr) { + // Need for const_cast is a consequence of the type_info::init_instance type: + // void (*init_instance)(instance *, const void *); + auto *holder_void_ptr = const_cast(holder_const_void_ptr); + + auto v_h = inst->get_value_and_holder(detail::get_type_info(typeid(type))); + if (!v_h.instance_registered()) { + register_instance(inst, v_h.value_ptr(), v_h.type); + v_h.set_instance_registered(); + } + auto *uninitialized_location = std::addressof(v_h.holder()); + auto *value_ptr_w_t = v_h.value_ptr(); + // Try downcast from `type` to `type_alias`: + inst->is_alias + = detail::dynamic_raw_ptr_cast_if_possible(value_ptr_w_t) != nullptr; + if (holder_void_ptr) { + // Note: inst->owned ignored. + auto *holder_ptr = static_cast(holder_void_ptr); + new (uninitialized_location) holder_type(std::move(*holder_ptr)); + } else if (!try_initialization_using_shared_from_this( + uninitialized_location, value_ptr_w_t, value_ptr_w_t)) { + if (inst->owned) { + new (uninitialized_location) holder_type(holder_type::from_raw_ptr_take_ownership( + value_ptr_w_t, /*void_cast_raw_ptr*/ inst->is_alias)); + } else { + new (uninitialized_location) + holder_type(holder_type::from_raw_ptr_unowned(value_ptr_w_t)); + } + } + v_h.set_holder_constructed(); + } + + // Deallocates an instance; via holder, if constructed; otherwise via operator delete. + // NOTE: The Python error indicator needs to cleared BEFORE this function is called. + // This is because we could be deallocating while cleaning up after a Python exception. + // If the error indicator is not cleared but the C++ destructor code makes Python C API + // calls, those calls are likely to generate a new exception, and pybind11 will then + // throw `error_already_set` from the C++ destructor. This is forbidden and will + // trigger std::terminate(). + static void dealloc_impl(detail::value_and_holder &v_h) { if (v_h.holder_constructed()) { v_h.holder().~holder_type(); v_h.set_holder_constructed(false); @@ -1878,6 +2885,32 @@ class class_ : public detail::generic_type { v_h.value_ptr() = nullptr; } + static void dealloc_without_manipulating_gil(detail::value_and_holder &v_h) { + error_scope scope; + dealloc_impl(v_h); + } + + static void dealloc_release_gil_before_calling_cpp_dtor(detail::value_and_holder &v_h) { + error_scope scope; + // Intentionally not using `gil_scoped_release` because the non-simple + // version unconditionally calls `get_internals()`. + // `Py_BEGIN_ALLOW_THREADS`, `Py_END_ALLOW_THREADS` cannot be used + // because those macros include `{` and `}`. + PyThreadState *py_ts = PyEval_SaveThread(); + try { + dealloc_impl(v_h); + } catch (...) { + // This code path is expected to be unreachable unless there is a + // bug in pybind11 itself. + // An alternative would be to mark this function, or + // `dealloc_impl()`, with `nothrow`, but that would be a subtle + // behavior change and could make debugging more difficult. + PyEval_RestoreThread(py_ts); + throw; + } + PyEval_RestoreThread(py_ts); + } + static detail::function_record *get_function_record(handle h) { h = detail::get_function(h); if (!h) { @@ -1888,17 +2921,15 @@ class class_ : public detail::generic_type { if (!func_self) { throw error_already_set(); } - if (!isinstance(func_self)) { - return nullptr; - } - auto cap = reinterpret_borrow(func_self); - if (!detail::is_function_record_capsule(cap)) { - return nullptr; - } - return cap.get_pointer(); + return detail::function_record_ptr_from_PyObject(func_self.ptr()); } }; +// Supports easier switching between py::class_ and py::class_: +// users can simply replace the `_` in `class_` with `h` or vice versa. +template +using classh = class_; + /// Binds an existing constructor taking arguments Args... template detail::initimpl::constructor init() { @@ -1935,7 +2966,7 @@ detail::initimpl::pickle_factory pickle(GetState &&g, SetSta PYBIND11_NAMESPACE_BEGIN(detail) inline str enum_name(handle arg) { - dict entries = arg.get_type().attr("__entries"); + dict entries = type::handle_of(arg).attr("__entries"); for (auto kv : entries) { if (handle(kv.second[int_(0)]).equal(arg)) { return pybind11::str(kv.first); @@ -1949,8 +2980,9 @@ struct enum_base { PYBIND11_NOINLINE void init(bool is_arithmetic, bool is_convertible) { m_base.attr("__entries") = dict(); - auto property = handle((PyObject *) &PyProperty_Type); - auto static_property = handle((PyObject *) get_internals().static_property_type); + auto property = handle(reinterpret_cast(&PyProperty_Type)); + auto static_property + = handle(reinterpret_cast(get_internals().static_property_type)); m_base.attr("__repr__") = cpp_function( [](const object &arg) -> str { @@ -1960,17 +2992,20 @@ struct enum_base { .format(std::move(type_name), enum_name(arg), int_(arg)); }, name("__repr__"), - is_method(m_base)); + is_method(m_base), + pos_only()); - m_base.attr("name") = property(cpp_function(&enum_name, name("name"), is_method(m_base))); + m_base.attr("name") + = property(cpp_function(&enum_name, name("name"), is_method(m_base), pos_only())); m_base.attr("__str__") = cpp_function( [](handle arg) -> str { object type_name = type::handle_of(arg).attr("__name__"); return pybind11::str("{}.{}").format(std::move(type_name), enum_name(arg)); }, - name("name"), - is_method(m_base)); + name("__str__"), + is_method(m_base), + pos_only()); if (options::show_enum_members_docstring()) { m_base.attr("__doc__") = static_property( @@ -1978,7 +3013,7 @@ struct enum_base { [](handle arg) -> std::string { std::string docstring; dict entries = arg.attr("__entries"); - if (((PyTypeObject *) arg.ptr())->tp_doc) { + if ((reinterpret_cast(arg.ptr()))->tp_doc) { docstring += std::string( reinterpret_cast(arg.ptr())->tp_doc); docstring += "\n\n"; @@ -2025,7 +3060,8 @@ struct enum_base { }, \ name(op), \ is_method(m_base), \ - arg("other")) + arg("other"), \ + pos_only()) #define PYBIND11_ENUM_OP_CONV(op, expr) \ m_base.attr(op) = cpp_function( \ @@ -2035,7 +3071,8 @@ struct enum_base { }, \ name(op), \ is_method(m_base), \ - arg("other")) + arg("other"), \ + pos_only()) #define PYBIND11_ENUM_OP_CONV_LHS(op, expr) \ m_base.attr(op) = cpp_function( \ @@ -2045,39 +3082,16 @@ struct enum_base { }, \ name(op), \ is_method(m_base), \ - arg("other")) + arg("other"), \ + pos_only()) if (is_convertible) { - PYBIND11_ENUM_OP_CONV_LHS("__eq__", !b.is_none() && a.equal(b)); - PYBIND11_ENUM_OP_CONV_LHS("__ne__", b.is_none() || !a.equal(b)); - if (is_arithmetic) { - PYBIND11_ENUM_OP_CONV("__lt__", a < b); - PYBIND11_ENUM_OP_CONV("__gt__", a > b); - PYBIND11_ENUM_OP_CONV("__le__", a <= b); - PYBIND11_ENUM_OP_CONV("__ge__", a >= b); - PYBIND11_ENUM_OP_CONV("__and__", a & b); - PYBIND11_ENUM_OP_CONV("__rand__", a & b); - PYBIND11_ENUM_OP_CONV("__or__", a | b); - PYBIND11_ENUM_OP_CONV("__ror__", a | b); - PYBIND11_ENUM_OP_CONV("__xor__", a ^ b); - PYBIND11_ENUM_OP_CONV("__rxor__", a ^ b); m_base.attr("__invert__") = cpp_function([](const object &arg) { return ~(int_(arg)); }, name("__invert__"), - is_method(m_base)); - } - } else { - PYBIND11_ENUM_OP_STRICT("__eq__", int_(a).equal(int_(b)), return false); - PYBIND11_ENUM_OP_STRICT("__ne__", !int_(a).equal(int_(b)), return true); - - if (is_arithmetic) { -#define PYBIND11_THROW throw type_error("Expected an enumeration of matching type!"); - PYBIND11_ENUM_OP_STRICT("__lt__", int_(a) < int_(b), PYBIND11_THROW); - PYBIND11_ENUM_OP_STRICT("__gt__", int_(a) > int_(b), PYBIND11_THROW); - PYBIND11_ENUM_OP_STRICT("__le__", int_(a) <= int_(b), PYBIND11_THROW); - PYBIND11_ENUM_OP_STRICT("__ge__", int_(a) >= int_(b), PYBIND11_THROW); -#undef PYBIND11_THROW + is_method(m_base), + pos_only()); } } @@ -2085,18 +3099,22 @@ struct enum_base { #undef PYBIND11_ENUM_OP_CONV #undef PYBIND11_ENUM_OP_STRICT - m_base.attr("__getstate__") = cpp_function( - [](const object &arg) { return int_(arg); }, name("__getstate__"), is_method(m_base)); + m_base.attr("__getstate__") = cpp_function([](const object &arg) { return int_(arg); }, + name("__getstate__"), + is_method(m_base), + pos_only()); - m_base.attr("__hash__") = cpp_function( - [](const object &arg) { return int_(arg); }, name("__hash__"), is_method(m_base)); + m_base.attr("__hash__") = cpp_function([](const object &arg) { return int_(arg); }, + name("__hash__"), + is_method(m_base), + pos_only()); } PYBIND11_NOINLINE void value(char const *name_, object value, const char *doc = nullptr) { dict entries = m_base.attr("__entries"); str name(name_); if (entries.contains(name)) { - std::string type_name = (std::string) str(m_base.attr("__name__")); + std::string type_name = std::string(str(m_base.attr("__name__"))); throw value_error(std::move(type_name) + ": element \"" + std::string(name_) + "\" already exists!"); } @@ -2163,7 +3181,6 @@ class enum_ : public class_ { public: using Base = class_; using Base::attr; - using Base::def; using Base::def_property_readonly; using Base::def_property_readonly_static; using Underlying = typename std::underlying_type::type; @@ -2176,14 +3193,85 @@ class enum_ : public class_ { template enum_(const handle &scope, const char *name, const Extra &...extra) : class_(scope, name, extra...), m_base(*this, scope) { + { + if (detail::global_internals_native_enum_type_map_contains( + std::type_index(typeid(Type)))) { + pybind11_fail("pybind11::enum_ \"" + std::string(name) + + "\" is already registered as a pybind11::native_enum!"); + } + } + constexpr bool is_arithmetic = detail::any_of...>::value; constexpr bool is_convertible = std::is_convertible::value; m_base.init(is_arithmetic, is_convertible); def(init([](Scalar i) { return static_cast(i); }), arg("value")); - def_property_readonly("value", [](Type value) { return (Scalar) value; }); - def("__int__", [](Type value) { return (Scalar) value; }); - def("__index__", [](Type value) { return (Scalar) value; }); + def_property_readonly("value", [](Type value) { return (Scalar) value; }, pos_only()); +#define PYBIND11_ENUM_OP_SAME_TYPE(op, expr) \ + def(op, [](Type a, Type b) { return expr; }, pybind11::name(op), arg("other"), pos_only()) +#define PYBIND11_ENUM_OP_SAME_TYPE_RHS_MAY_BE_NONE(op, expr) \ + def(op, [](Type a, Type *b_ptr) { return expr; }, pybind11::name(op), arg("other"), pos_only()) +#define PYBIND11_ENUM_OP_SCALAR(op, op_expr) \ + def( \ + op, \ + [](Type a, Scalar b) { return static_cast(a) op_expr b; }, \ + pybind11::name(op), \ + arg("other"), \ + pos_only()) +#define PYBIND11_ENUM_OP_CONV_ARITHMETIC(op, op_expr) \ + /* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \ + PYBIND11_ENUM_OP_SAME_TYPE(op, static_cast(a) op_expr static_cast(b)); \ + PYBIND11_ENUM_OP_SCALAR(op, op_expr) +#define PYBIND11_ENUM_OP_REJECT_UNRELATED_TYPE(op, strict_behavior) \ + def( \ + op, \ + [](Type, const object &) { strict_behavior; }, \ + pybind11::name(op), \ + arg("other"), \ + pos_only()) +#define PYBIND11_ENUM_OP_STRICT_ARITHMETIC(op, op_expr, strict_behavior) \ + /* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \ + PYBIND11_ENUM_OP_SAME_TYPE(op, static_cast(a) op_expr static_cast(b)); \ + PYBIND11_ENUM_OP_REJECT_UNRELATED_TYPE(op, strict_behavior); + + PYBIND11_ENUM_OP_SAME_TYPE_RHS_MAY_BE_NONE("__eq__", b_ptr && a == *b_ptr); + PYBIND11_ENUM_OP_SAME_TYPE_RHS_MAY_BE_NONE("__ne__", !b_ptr || a != *b_ptr); + if (std::is_convertible::value) { + PYBIND11_ENUM_OP_SCALAR("__eq__", ==); + PYBIND11_ENUM_OP_SCALAR("__ne__", !=); + if (is_arithmetic) { + PYBIND11_ENUM_OP_CONV_ARITHMETIC("__lt__", <); + PYBIND11_ENUM_OP_CONV_ARITHMETIC("__gt__", >); + PYBIND11_ENUM_OP_CONV_ARITHMETIC("__le__", <=); + PYBIND11_ENUM_OP_CONV_ARITHMETIC("__ge__", >=); + PYBIND11_ENUM_OP_CONV_ARITHMETIC("__and__", &); + PYBIND11_ENUM_OP_CONV_ARITHMETIC("__rand__", &); + PYBIND11_ENUM_OP_CONV_ARITHMETIC("__or__", |); + PYBIND11_ENUM_OP_CONV_ARITHMETIC("__ror__", |); + PYBIND11_ENUM_OP_CONV_ARITHMETIC("__xor__", ^); + PYBIND11_ENUM_OP_CONV_ARITHMETIC("__rxor__", ^); + } + } else if (is_arithmetic) { +#define PYBIND11_ENUM_OP_THROW_TYPE_ERROR \ + throw type_error("Expected an enumeration of matching type!"); + PYBIND11_ENUM_OP_STRICT_ARITHMETIC("__lt__", <, PYBIND11_ENUM_OP_THROW_TYPE_ERROR); + PYBIND11_ENUM_OP_STRICT_ARITHMETIC("__gt__", >, PYBIND11_ENUM_OP_THROW_TYPE_ERROR); + PYBIND11_ENUM_OP_STRICT_ARITHMETIC("__le__", <=, PYBIND11_ENUM_OP_THROW_TYPE_ERROR); + PYBIND11_ENUM_OP_STRICT_ARITHMETIC("__ge__", >=, PYBIND11_ENUM_OP_THROW_TYPE_ERROR); +#undef PYBIND11_ENUM_OP_THROW_TYPE_ERROR + } + PYBIND11_ENUM_OP_REJECT_UNRELATED_TYPE("__eq__", return false); + PYBIND11_ENUM_OP_REJECT_UNRELATED_TYPE("__ne__", return true); + +#undef PYBIND11_ENUM_OP_SAME_TYPE +#undef PYBIND11_ENUM_OP_SAME_TYPE_RHS_MAY_BE_NONE +#undef PYBIND11_ENUM_OP_SCALAR +#undef PYBIND11_ENUM_OP_CONV_ARITHMETIC +#undef PYBIND11_ENUM_OP_REJECT_UNRELATED_TYPE +#undef PYBIND11_ENUM_OP_STRICT_ARITHMETIC + + def("__int__", [](Type value) { return (Scalar) value; }, pos_only()); + def("__index__", [](Type value) { return (Scalar) value; }, pos_only()); attr("__setstate__") = cpp_function( [](detail::value_and_holder &v_h, Scalar arg) { detail::initimpl::setstate( @@ -2192,7 +3280,30 @@ class enum_ : public class_ { detail::is_new_style_constructor(), pybind11::name("__setstate__"), is_method(*this), - arg("state")); + arg("state"), + pos_only()); + } + + template + enum_ &def(const char *name_, Func &&f, const Extra &...extra) { + if (std::strcmp(name_, "__str__") == 0) { + Base::def(name_, std::forward(f), prepend{}, extra...); + } else { + Base::def(name_, std::forward(f), extra...); + } + return *this; + } + + // Avoid using Base::def here: GCC 15/MinGW sees the duplicate dependent-base + // def(const char *, ...) template as ambiguous with enum_::def(const char *, ...). + template ::type, const char *>::value, + int> = 0> + enum_ &def(T &&op, const Extra &...extra) { + Base::def(std::forward(op), extra...); + return *this; } /// Export enumeration entries into the parent scope @@ -2263,28 +3374,39 @@ keep_alive_impl(size_t Nurse, size_t Patient, function_call &call, handle ret) { inline std::pair all_type_info_get_cache(PyTypeObject *type) { - auto res = get_internals() - .registered_types_py + auto res = with_internals([type](internals &internals) { + auto ins = internals + .registered_types_py #ifdef __cpp_lib_unordered_map_try_emplace - .try_emplace(type); + .try_emplace(type); #else - .emplace(type, std::vector()); + .emplace(type, std::vector()); #endif + if (ins.second) { + // For free-threading mode, this call must be under + // the with_internals() mutex lock, to avoid that other threads + // continue running with the empty ins.first->second. + all_type_info_populate(type, ins.first->second); + } + return ins; + }); if (res.second) { // New cache entry created; set up a weak reference to automatically remove it if the type // gets destroyed: - weakref((PyObject *) type, cpp_function([type](handle wr) { - get_internals().registered_types_py.erase(type); - - // TODO consolidate the erasure code in pybind11_meta_dealloc() in class.h - auto &cache = get_internals().inactive_override_cache; - for (auto it = cache.begin(), last = cache.end(); it != last;) { - if (it->first == reinterpret_cast(type)) { - it = cache.erase(it); - } else { - ++it; + weakref(reinterpret_cast(type), cpp_function([type](handle wr) { + with_internals([type](internals &internals) { + internals.registered_types_py.erase(type); + + // TODO consolidate the erasure code in pybind11_meta_dealloc() in class.h + auto &cache = internals.inactive_override_cache; + for (auto it = cache.begin(), last = cache.end(); it != last;) { + if (it->first == reinterpret_cast(type)) { + it = cache.erase(it); + } else { + ++it; + } } - } + }); wr.dec_ref(); })) @@ -2361,13 +3483,20 @@ template +// NOLINTNEXTLINE(performance-unnecessary-value-param) iterator make_iterator_impl(Iterator first, Sentinel last, Extra &&...extra) { using state = detail::iterator_state; // TODO: state captures only the types of Extra, not the values + // For Python < 3.14.0rc1, pycritical_section uses direct mutex locking (same as a unique + // lock), which may deadlock during type registration. See detail/internals.h for details. +#if PY_VERSION_HEX >= 0x030E00C1 // 3.14.0rc1 + PYBIND11_LOCK_INTERNALS(get_internals()); +#endif if (!detail::get_type_info(typeid(state), false)) { class_(handle(), "iterator", pybind11::module_local()) - .def("__iter__", [](state &s) -> state & { return s; }) + .def( + "__iter__", [](state &s) -> state & { return s; }, pos_only()) .def( "__next__", [](state &s) -> ValueType { @@ -2384,10 +3513,11 @@ iterator make_iterator_impl(Iterator first, Sentinel last, Extra &&...extra) { // NOLINTNEXTLINE(readability-const-return-type) // PR #3263 }, std::forward(extra)..., + pos_only(), Policy); } - return cast(state{first, last, true}); + return cast(state{std::forward(first), std::forward(last), true}); } PYBIND11_NAMESPACE_END(detail) @@ -2398,13 +3528,16 @@ template ::result_type, typename... Extra> -iterator make_iterator(Iterator first, Sentinel last, Extra &&...extra) { +// NOLINTNEXTLINE(performance-unnecessary-value-param) +typing::Iterator make_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, Sentinel, ValueType, - Extra...>(first, last, std::forward(extra)...); + Extra...>(std::forward(first), + std::forward(last), + std::forward(extra)...); } /// Makes a python iterator over the keys (`.first`) of a iterator over pairs from a @@ -2414,13 +3547,15 @@ template ::result_type, typename... Extra> -iterator make_key_iterator(Iterator first, Sentinel last, Extra &&...extra) { +typing::Iterator make_key_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, Sentinel, KeyType, - Extra...>(first, last, std::forward(extra)...); + Extra...>(std::forward(first), + std::forward(last), + std::forward(extra)...); } /// Makes a python iterator over the values (`.second`) of a iterator over pairs from a @@ -2430,21 +3565,25 @@ template ::result_type, typename... Extra> -iterator make_value_iterator(Iterator first, Sentinel last, Extra &&...extra) { +typing::Iterator make_value_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, Sentinel, ValueType, - Extra...>(first, last, std::forward(extra)...); + Extra...>(std::forward(first), + std::forward(last), + std::forward(extra)...); } /// Makes an iterator over values of an stl container or other container supporting /// `std::begin()`/`std::end()` template ()))>::result_type, typename... Extra> -iterator make_iterator(Type &value, Extra &&...extra) { +typing::Iterator make_iterator(Type &value, Extra &&...extra) { return make_iterator( std::begin(value), std::end(value), std::forward(extra)...); } @@ -2453,8 +3592,10 @@ iterator make_iterator(Type &value, Extra &&...extra) { /// `std::begin()`/`std::end()` template ()))>::result_type, typename... Extra> -iterator make_key_iterator(Type &value, Extra &&...extra) { +typing::Iterator make_key_iterator(Type &value, Extra &&...extra) { return make_key_iterator( std::begin(value), std::end(value), std::forward(extra)...); } @@ -2463,8 +3604,10 @@ iterator make_key_iterator(Type &value, Extra &&...extra) { /// `std::begin()`/`std::end()` template ()))>::result_type, typename... Extra> -iterator make_value_iterator(Type &value, Extra &&...extra) { +typing::Iterator make_value_iterator(Type &value, Extra &&...extra) { return make_value_iterator( std::begin(value), std::end(value), std::forward(extra)...); } @@ -2475,9 +3618,15 @@ void implicitly_convertible() { bool &flag; explicit set_flag(bool &flag_) : flag(flag_) { flag_ = true; } ~set_flag() { flag = false; } + + // Prevent copying/moving to ensure RAII guard is used safely + set_flag(const set_flag &) = delete; + set_flag(set_flag &&) = delete; + set_flag &operator=(const set_flag &) = delete; + set_flag &operator=(set_flag &&) = delete; }; auto implicit_caster = [](PyObject *obj, PyTypeObject *type) -> PyObject * { - static bool currently_used = false; + thread_local bool currently_used = false; if (currently_used) { // implicit conversions are non-reentrant return nullptr; } @@ -2487,7 +3636,7 @@ void implicitly_convertible() { } tuple args(1); args[0] = obj; - PyObject *result = PyObject_Call((PyObject *) type, args.ptr(), nullptr); + PyObject *result = PyObject_Call(reinterpret_cast(type), args.ptr(), nullptr); if (result == nullptr) { PyErr_Clear(); } @@ -2502,8 +3651,12 @@ void implicitly_convertible() { } inline void register_exception_translator(ExceptionTranslator &&translator) { - detail::get_internals().registered_exception_translators.push_front( - std::forward(translator)); + detail::with_exception_translators( + [&](std::forward_list &exception_translators, + std::forward_list &local_exception_translators) { + (void) local_exception_translators; + exception_translators.push_front(std::forward(translator)); + }); } /** @@ -2513,14 +3666,18 @@ inline void register_exception_translator(ExceptionTranslator &&translator) { * the exception. */ inline void register_local_exception_translator(ExceptionTranslator &&translator) { - detail::get_local_internals().registered_exception_translators.push_front( - std::forward(translator)); + detail::with_exception_translators( + [&](std::forward_list &exception_translators, + std::forward_list &local_exception_translators) { + (void) exception_translators; + local_exception_translators.push_front(std::forward(translator)); + }); } /** * Wrapper to generate a new Python exception type. * - * This should only be used with PyErr_SetString for now. + * This should only be used with py::set_error() for now. * It is not (yet) possible to use as a py::base. * Template type argument is reserved for future use. */ @@ -2541,27 +3698,25 @@ class exception : public object { } // Sets the current python exception to this exception object with the given message - void operator()(const char *message) { PyErr_SetString(m_ptr, message); } + PYBIND11_DEPRECATED("Please use py::set_error() instead " + "(https://github.com/pybind/pybind11/pull/4772)") + void operator()(const char *message) const { set_error(*this, message); } }; PYBIND11_NAMESPACE_BEGIN(detail) -// Returns a reference to a function-local static exception object used in the simple -// register_exception approach below. (It would be simpler to have the static local variable -// directly in register_exception, but that makes clang <3.5 segfault - issue #1349). -template -exception &get_exception_object() { - static exception ex; - return ex; -} + +template <> +struct handle_type_name> { + static constexpr auto name = const_name("Exception"); +}; // Helper function for register_exception and register_local_exception template exception & register_exception_impl(handle scope, const char *name, handle base, bool isLocal) { - auto &ex = detail::get_exception_object(); - if (!ex) { - ex = exception(scope, name, base); - } + PYBIND11_CONSTINIT static gil_safe_call_once_and_store> exc_storage; + exc_storage.call_once_and_store_result( + [&]() { return exception(scope, name, base); }); auto register_func = isLocal ? ®ister_local_exception_translator : ®ister_exception_translator; @@ -2573,10 +3728,10 @@ register_exception_impl(handle scope, const char *name, handle base, bool isLoca try { std::rethrow_exception(p); } catch (const CppException &e) { - detail::get_exception_object()(e.what()); + set_error(exc_storage.get_stored(), e.what()); } }); - return ex; + return exc_storage.get_stored(); } PYBIND11_NAMESPACE_END(detail) @@ -2609,34 +3764,20 @@ register_local_exception(handle scope, const char *name, handle base = PyExc_Exc PYBIND11_NAMESPACE_BEGIN(detail) PYBIND11_NOINLINE void print(const tuple &args, const dict &kwargs) { - auto strings = tuple(args.size()); - for (size_t i = 0; i < args.size(); ++i) { - strings[i] = str(args[i]); - } - auto sep = kwargs.contains("sep") ? kwargs["sep"] : str(" "); - auto line = sep.attr("join")(std::move(strings)); - - object file; - if (kwargs.contains("file")) { - file = kwargs["file"].cast(); - } else { - try { - file = module_::import("sys").attr("stdout"); - } catch (const error_already_set &) { - /* If print() is called from code that is executed as - part of garbage collection during interpreter shutdown, - importing 'sys' can fail. Give up rather than crashing the - interpreter in this case. */ - return; - } +#if PY_VERSION_HEX >= 0x030D0000 + auto builtins = reinterpret_steal(PyEval_GetFrameBuiltins()); +#else + auto builtins = reinterpret_borrow(PyEval_GetBuiltins()); +#endif + // The builtins dictionary may already be partially cleared during interpreter shutdown. + auto native_print = reinterpret_steal(dict_getitemstringref(builtins.ptr(), "print")); + if (!native_print) { + return; } - - auto write = file.attr("write"); - write(std::move(line)); - write(kwargs.contains("end") ? kwargs["end"] : str("\n")); - - if (kwargs.contains("flush") && kwargs["flush"].cast()) { - file.attr("flush")(); + auto result + = reinterpret_steal(PyObject_Call(native_print.ptr(), args.ptr(), kwargs.ptr())); + if (!result) { + throw error_already_set(); } } PYBIND11_NAMESPACE_END(detail) @@ -2673,32 +3814,46 @@ get_type_override(const void *this_ptr, const type_info *this_type, const char * /* Cache functions that aren't overridden in Python to avoid many costly Python dictionary lookups below */ - auto &cache = get_internals().inactive_override_cache; - if (cache.find(key) != cache.end()) { + bool not_overridden = with_internals([&key](internals &internals) { + auto &cache = internals.inactive_override_cache; + return cache.find(key) != cache.end(); + }); + if (not_overridden) { return function(); } function override = getattr(self, name, function()); if (override.is_cpp_function()) { - cache.insert(std::move(key)); + with_internals([&](internals &internals) { + internals.inactive_override_cache.insert(std::move(key)); + }); return function(); } /* Don't call dispatch code if invoked from overridden function. - Unfortunately this doesn't work on PyPy. */ -#if !defined(PYPY_VERSION) -# if PY_VERSION_HEX >= 0x03090000 + Unfortunately this doesn't work on PyPy and GraalPy. */ +#if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) PyFrameObject *frame = PyThreadState_GetFrame(PyThreadState_Get()); if (frame != nullptr) { PyCodeObject *f_code = PyFrame_GetCode(frame); // f_code is guaranteed to not be NULL - if ((std::string) str(f_code->co_name) == name && f_code->co_argcount > 0) { + if (std::string(str(f_code->co_name)) == name && f_code->co_argcount > 0) { +# if PY_VERSION_HEX >= 0x030d0000 + PyObject *locals = PyEval_GetFrameLocals(); +# else PyObject *locals = PyEval_GetLocals(); + Py_XINCREF(locals); +# endif if (locals != nullptr) { +# if PY_VERSION_HEX >= 0x030b0000 + PyObject *co_varnames = PyCode_GetVarnames(f_code); +# else PyObject *co_varnames = PyObject_GetAttrString((PyObject *) f_code, "co_varnames"); +# endif PyObject *self_arg = PyTuple_GET_ITEM(co_varnames, 0); Py_DECREF(co_varnames); PyObject *self_caller = dict_getitem(locals, self_arg); + Py_DECREF(locals); if (self_caller == self.ptr()) { Py_DECREF(f_code); Py_DECREF(frame); @@ -2709,18 +3864,6 @@ get_type_override(const void *this_ptr, const type_info *this_type, const char * Py_DECREF(f_code); Py_DECREF(frame); } -# else - PyFrameObject *frame = PyThreadState_Get()->frame; - if (frame != nullptr && (std::string) str(frame->f_code->co_name) == name - && frame->f_code->co_argcount > 0) { - PyFrame_FastToLocals(frame); - PyObject *self_caller - = dict_getitem(frame->f_locals, PyTuple_GET_ITEM(frame->f_code->co_varnames, 0)); - if (self_caller == self.ptr()) { - return function(); - } - } -# endif #else /* PyPy currently doesn't provide a detailed cpyext emulation of @@ -2775,11 +3918,17 @@ function get_override(const T *this_ptr, const char *name) { = pybind11::get_override(static_cast(this), name); \ if (override) { \ auto o = override(__VA_ARGS__); \ - if (pybind11::detail::cast_is_temporary_value_reference::value) { \ + PYBIND11_WARNING_PUSH \ + PYBIND11_WARNING_DISABLE_MSVC(4127) \ + if PYBIND11_MAYBE_CONSTEXPR ( \ + pybind11::detail::cast_is_temporary_value_reference::value \ + && !pybind11::detail::is_same_ignoring_cvref::value) { \ static pybind11::detail::override_caster_t caster; \ return pybind11::detail::cast_ref(std::move(o), caster); \ + } else { \ + return pybind11::detail::cast_safe(std::move(o)); \ } \ - return pybind11::detail::cast_safe(std::move(o)); \ + PYBIND11_WARNING_POP \ } \ } while (false) diff --git a/lib/pybind11/include/pybind11/pytypes.h b/lib/pybind11/include/pybind11/pytypes.h index f11ed5d..13a6eba 100644 --- a/lib/pybind11/include/pybind11/pytypes.h +++ b/lib/pybind11/include/pybind11/pytypes.h @@ -48,6 +48,9 @@ PYBIND11_NAMESPACE_BEGIN(detail) class args_proxy; bool isinstance_generic(handle obj, const std::type_info &tp); +template +bool isinstance_native_enum(handle obj, const std::type_info &tp); + // Accessor forward declarations template class accessor; @@ -59,6 +62,7 @@ struct sequence_item; struct list_item; struct tuple_item; } // namespace accessor_policies +// PLEASE KEEP handle_type_name SPECIALIZATIONS IN SYNC. using obj_attr_accessor = accessor; using str_attr_accessor = accessor; using item_accessor = accessor; @@ -77,7 +81,9 @@ using is_pyobject = std::is_base_of>; \endrst */ template class object_api : public pyobject_tag { + object_api() = default; const Derived &derived() const { return static_cast(*this); } + friend Derived; public: /** \rst @@ -112,6 +118,17 @@ class object_api : public pyobject_tag { /// See above (the only difference is that the key is provided as a string literal) str_attr_accessor attr(const char *key) const; + /** \rst + Similar to the above attr functions with the difference that the templated Type + is used to set the `__annotations__` dict value to the corresponding key. Worth noting + that attr_with_type_hint is implemented in cast.h. + \endrst */ + template + obj_attr_accessor attr_with_type_hint(handle key) const; + /// See above (the only difference is that the key is provided as a string literal) + template + str_attr_accessor attr_with_type_hint(const char *key) const; + /** \rst Matches * unpacking in Python, e.g. to unpack arguments out of a ``tuple`` or ``list`` for a function call. Applying another * to the result yields @@ -181,11 +198,21 @@ class object_api : public pyobject_tag { /// Get or set the object's docstring, i.e. ``obj.__doc__``. str_attr_accessor doc() const; + /// Get or set the object's annotations, i.e. ``obj.__annotations__``. + object annotations() const; + /// Return the object's current reference count - int ref_count() const { return static_cast(Py_REFCNT(derived().ptr())); } + ssize_t ref_count() const { +#ifdef PYPY_VERSION + // PyPy uses the top few bits for REFCNT_FROM_PYPY & REFCNT_FROM_PYPY_LIGHT + // Following pybind11 2.12.1 and older behavior and removing this part + return static_cast(static_cast(Py_REFCNT(derived().ptr()))); +#else + return Py_REFCNT(derived().ptr()); +#endif + } - // TODO PYBIND11_DEPRECATED( - // "Call py::type::handle_of(h) or py::type::of(h) instead of h.get_type()") + PYBIND11_DEPRECATED("Call py::type::handle_of(h) or py::type::of(h) instead of h.get_type()") handle get_type() const; private: @@ -232,8 +259,7 @@ class handle : public detail::object_api { detail::enable_if_t, detail::is_pyobj_ptr_or_nullptr_t>, std::is_convertible>::value, - int> - = 0> + int> = 0> // NOLINTNEXTLINE(google-explicit-constructor) handle(T &obj) : m_ptr(obj) {} @@ -251,7 +277,7 @@ class handle : public detail::object_api { inc_ref_counter(1); #endif #ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF - if (m_ptr != nullptr && !PyGILState_Check()) { + if (m_ptr != nullptr && PyGILState_Check() == 0) { throw_gilstate_error("pybind11::handle::inc_ref()"); } #endif @@ -266,7 +292,7 @@ class handle : public detail::object_api { \endrst */ const handle &dec_ref() const & { #ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF - if (m_ptr != nullptr && !PyGILState_Check()) { + if (m_ptr != nullptr && PyGILState_Check() == 0) { throw_gilstate_error("pybind11::handle::dec_ref()"); } #endif @@ -303,16 +329,21 @@ class handle : public detail::object_api { stderr, "%s is being called while the GIL is either not held or invalid. Please see " "https://pybind11.readthedocs.io/en/stable/advanced/" - "misc.html#common-sources-of-global-interpreter-lock-errors for debugging advice.\n", + "misc.html#common-sources-of-global-interpreter-lock-errors for debugging advice.\n" + "If you are convinced there is no bug in your code, you can #define " + "PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF " + "to disable this check. In that case you have to ensure this #define is consistently " + "used for all translation units linked into a given pybind11 extension, otherwise " + "there will be ODR violations.", function_name.c_str()); - fflush(stderr); if (Py_TYPE(m_ptr)->tp_name != nullptr) { fprintf(stderr, - "The failing %s call was triggered on a %s object.\n", + " The failing %s call was triggered on a %s object.", function_name.c_str(), Py_TYPE(m_ptr)->tp_name); - fflush(stderr); } + fprintf(stderr, "\n"); + fflush(stderr); throw std::runtime_error(function_name + " PyGILState_Check() failure."); } #endif @@ -329,6 +360,14 @@ class handle : public detail::object_api { #endif }; +inline void set_error(const handle &type, const char *message) { + PyErr_SetString(type.ptr(), message); +} + +inline void set_error(const handle &type, const handle &value) { + PyErr_SetObject(type.ptr(), value.ptr()); +} + /** \rst Holds a reference to a Python object (with reference counting) @@ -471,13 +510,24 @@ inline const char *obj_class_name(PyObject *obj) { std::string error_string(); +// The code in this struct is very unusual, to minimize the chances of +// masking bugs (elsewhere) by errors during the error handling (here). +// This is meant to be a lifeline for troubleshooting long-running processes +// that crash under conditions that are virtually impossible to reproduce. +// Low-level implementation alternatives are preferred to higher-level ones +// that might raise cascading exceptions. Last-ditch-kind-of attempts are made +// to report as much of the original error as possible, even if there are +// secondary issues obtaining some of the details. struct error_fetch_and_normalize { - // Immediate normalization is long-established behavior (starting with - // https://github.com/pybind/pybind11/commit/135ba8deafb8bf64a15b24d1513899eb600e2011 - // from Sep 2016) and safest. Normalization could be deferred, but this could mask - // errors elsewhere, the performance gain is very minor in typical situations - // (usually the dominant bottleneck is EH unwinding), and the implementation here - // would be more complex. + // This comment only applies to Python <= 3.11: + // Immediate normalization is long-established behavior (starting with + // https://github.com/pybind/pybind11/commit/135ba8deafb8bf64a15b24d1513899eb600e2011 + // from Sep 2016) and safest. Normalization could be deferred, but this could mask + // errors elsewhere, the performance gain is very minor in typical situations + // (usually the dominant bottleneck is EH unwinding), and the implementation here + // would be more complex. + // Starting with Python 3.12, PyErr_Fetch() normalizes exceptions immediately. + // Any errors during normalization are tracked under __notes__. explicit error_fetch_and_normalize(const char *called) { PyErr_Fetch(&m_type.ptr(), &m_value.ptr(), &m_trace.ptr()); if (!m_type) { @@ -492,6 +542,19 @@ struct error_fetch_and_normalize { "of the original active exception type."); } m_lazy_error_string = exc_type_name_orig; +#if PY_VERSION_HEX >= 0x030C0000 + // The presence of __notes__ is likely due to exception normalization + // errors, although that is not necessarily true, therefore insert a + // hint only: + const int has_notes = PyObject_HasAttrString(m_value.ptr(), "__notes__"); + if (has_notes == 1) { + m_lazy_error_string += "[WITH __notes__]"; + } else if (has_notes == -1) { + // Ignore secondary errors when probing for __notes__ to avoid leaking a + // spurious exception while still reporting the original error. + PyErr_Clear(); + } +#else // PyErr_NormalizeException() may change the exception type if there are cascading // failures. This can potentially be extremely confusing. PyErr_NormalizeException(&m_type.ptr(), &m_value.ptr(), &m_trace.ptr()); @@ -506,12 +569,6 @@ struct error_fetch_and_normalize { + " failed to obtain the name " "of the normalized active exception type."); } -#if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x07030a00 - // This behavior runs the risk of masking errors in the error handling, but avoids a - // conflict with PyPy, which relies on the normalization here to change OSError to - // FileNotFoundError (https://github.com/pybind/pybind11/issues/4075). - m_lazy_error_string = exc_type_name_norm; -#else if (exc_type_name_norm != m_lazy_error_string) { std::string msg = std::string(called) + ": MISMATCH of original and normalized " @@ -558,6 +615,40 @@ struct error_fetch_and_normalize { } } } +#if PY_VERSION_HEX >= 0x030B0000 + auto notes + = reinterpret_steal(PyObject_GetAttrString(m_value.ptr(), "__notes__")); + if (!notes) { + PyErr_Clear(); // No notes is good news. + } else { + auto len_notes = PyList_Size(notes.ptr()); + if (len_notes < 0) { + result += "\nFAILURE obtaining len(__notes__): " + detail::error_string(); + } else { + result += "\n__notes__ (len=" + std::to_string(len_notes) + "):"; + for (ssize_t i = 0; i < len_notes; i++) { + PyObject *note = PyList_GET_ITEM(notes.ptr(), i); + auto note_bytes = reinterpret_steal( + PyUnicode_AsEncodedString(note, "utf-8", "backslashreplace")); + if (!note_bytes) { + result += "\nFAILURE obtaining __notes__[" + std::to_string(i) + + "]: " + detail::error_string(); + } else { + char *buffer = nullptr; + Py_ssize_t length = 0; + if (PyBytes_AsStringAndSize(note_bytes.ptr(), &buffer, &length) + == -1) { + result += "\nFAILURE formatting __notes__[" + std::to_string(i) + + "]: " + detail::error_string(); + } else { + result += '\n'; + result += std::string(buffer, static_cast(length)); + } + } + } + } + } +#endif } else { result = ""; } @@ -567,7 +658,7 @@ struct error_fetch_and_normalize { bool have_trace = false; if (m_trace) { -#if !defined(PYPY_VERSION) +#if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) auto *tb = reinterpret_cast(m_trace.ptr()); // Get the deepest trace possible. @@ -579,12 +670,7 @@ struct error_fetch_and_normalize { Py_XINCREF(frame); result += "\n\nAt:\n"; while (frame) { -# if PY_VERSION_HEX >= 0x030900B1 PyCodeObject *f_code = PyFrame_GetCode(frame); -# else - PyCodeObject *f_code = frame->f_code; - Py_INCREF(f_code); -# endif int lineno = PyFrame_GetLineNumber(frame); result += " "; result += handle(f_code->co_filename).cast(); @@ -594,12 +680,7 @@ struct error_fetch_and_normalize { result += handle(f_code->co_name).cast(); result += '\n'; Py_DECREF(f_code); -# if PY_VERSION_HEX >= 0x030900B1 auto *b_frame = PyFrame_GetBack(frame); -# else - auto *b_frame = frame->f_back; - Py_XINCREF(b_frame); -# endif Py_DECREF(frame); frame = b_frame; } @@ -769,11 +850,12 @@ bool isinstance(handle obj) { template ::value, int> = 0> bool isinstance(handle obj) { - return detail::isinstance_generic(obj, typeid(T)); + return detail::isinstance_native_enum(obj, typeid(T)) + || detail::isinstance_generic(obj, typeid(T)); } template <> -inline bool isinstance(handle) = delete; +bool isinstance(handle) = delete; template <> inline bool isinstance(handle obj) { return obj.ptr() != nullptr; @@ -904,6 +986,65 @@ inline PyObject *dict_getitem(PyObject *v, PyObject *key) { return rv; } +// PyDict_GetItemStringRef was added in Python 3.13.0a1. +// See also: https://github.com/python/pythoncapi-compat/blob/main/pythoncapi_compat.h +inline PyObject *dict_getitemstringref(PyObject *v, const char *key) { +#if PY_VERSION_HEX >= 0x030D00A1 + PyObject *rv = nullptr; + if (PyDict_GetItemStringRef(v, key, &rv) < 0) { + throw error_already_set(); + } + return rv; +#else + PyObject *rv = dict_getitemstring(v, key); + if (rv == nullptr && PyErr_Occurred()) { + throw error_already_set(); + } + Py_XINCREF(rv); + return rv; +#endif +} + +inline PyObject *dict_setdefaultstring(PyObject *v, const char *key, PyObject *defaultobj) { + PyObject *kv = PyUnicode_FromString(key); + if (kv == nullptr) { + throw error_already_set(); + } + + PyObject *rv = PyDict_SetDefault(v, kv, defaultobj); + Py_DECREF(kv); + if (rv == nullptr) { + throw error_already_set(); + } + return rv; +} + +// PyDict_SetDefaultRef was added in Python 3.13.0a4. +// See also: https://github.com/python/pythoncapi-compat/blob/main/pythoncapi_compat.h +inline PyObject *dict_setdefaultstringref(PyObject *v, const char *key, PyObject *defaultobj) { +#if PY_VERSION_HEX >= 0x030D00A4 + PyObject *kv = PyUnicode_FromString(key); + if (kv == nullptr) { + throw error_already_set(); + } + + PyObject *rv = nullptr; + if (PyDict_SetDefaultRef(v, kv, defaultobj, &rv) < 0) { + Py_DECREF(kv); + throw error_already_set(); + } + Py_DECREF(kv); + return rv; +#else + PyObject *rv = dict_setdefaultstring(v, key, defaultobj); + if (rv == nullptr || PyErr_Occurred()) { + throw error_already_set(); + } + Py_XINCREF(rv); + return rv; +#endif +} + // Helper aliases/functions to support implicit casting of values given to python // accessors/methods. When given a pyobject, this simply returns the pyobject as-is; for other C++ // type, the value goes through pybind11::cast(obj) to convert it to an `object`. @@ -934,11 +1075,11 @@ class accessor : public object_api> { void operator=(const accessor &a) & { operator=(handle(a)); } template - void operator=(T &&value) && { + enable_if_t>::value> operator=(T &&value) && { Policy::set(obj, key, object_or_cast(std::forward(value))); } template - void operator=(T &&value) & { + enable_if_t>::value> operator=(T &&value) & { get_cache() = ensure_object(object_or_cast(std::forward(value))); } @@ -1166,6 +1307,7 @@ class sequence_fast_readonly { using pointer = arrow_proxy; sequence_fast_readonly(handle obj, ssize_t n) : ptr(PySequence_Fast_ITEMS(obj.ptr()) + n) {} + sequence_fast_readonly() = default; // NOLINTNEXTLINE(readability-const-return-type) // PR #3263 reference dereference() const { return *ptr; } @@ -1188,6 +1330,7 @@ class sequence_slow_readwrite { using pointer = arrow_proxy; sequence_slow_readwrite(handle obj, ssize_t index) : obj(obj), index(index) {} + sequence_slow_readwrite() = default; reference dereference() const { return {obj, static_cast(index)}; } void increment() { ++index; } @@ -1261,7 +1404,7 @@ inline bool PyUnicode_Check_Permissive(PyObject *o) { # define PYBIND11_STR_CHECK_FUN PyUnicode_Check #endif -inline bool PyStaticMethod_Check(PyObject *o) { return o->ob_type == &PyStaticMethod_Type; } +inline bool PyStaticMethod_Check(PyObject *o) { return Py_TYPE(o) == &PyStaticMethod_Type; } class kwargs_proxy : public handle { public: @@ -1292,6 +1435,18 @@ class simple_collector; template class unpacking_collector; +inline object get_scope_module(handle scope) { + if (scope) { + if (hasattr(scope, "__module__")) { + return scope.attr("__module__"); + } + if (hasattr(scope, "__name__")) { + return scope.attr("__name__"); + } + } + return object(); +} + PYBIND11_NAMESPACE_END(detail) // TODO: After the deprecated constructors are removed, this macro can be simplified by @@ -1375,11 +1530,17 @@ class iterator : public object { PYBIND11_OBJECT_DEFAULT(iterator, object, PyIter_Check) iterator &operator++() { + init(); advance(); return *this; } iterator operator++(int) { + // Note: We must call init() first so that rv.value is + // the same as this->value just before calling advance(). + // Otherwise, dereferencing the returned iterator may call + // advance() again and return the 3rd item instead of the 1st. + init(); auto rv = *this; advance(); return rv; @@ -1387,15 +1548,12 @@ class iterator : public object { // NOLINTNEXTLINE(readability-const-return-type) // PR #3263 reference operator*() const { - if (m_ptr && !value.ptr()) { - auto &self = const_cast(*this); - self.advance(); - } + init(); return value; } pointer operator->() const { - operator*(); + init(); return &value; } @@ -1418,6 +1576,13 @@ class iterator : public object { friend bool operator!=(const iterator &a, const iterator &b) { return a->ptr() != b->ptr(); } private: + void init() const { + if (m_ptr && !value.ptr()) { + auto &self = const_cast(*this); + self.advance(); + } + } + void advance() { value = reinterpret_steal(PyIter_Next(m_ptr)); if (value.ptr() == nullptr && PyErr_Occurred()) { @@ -1426,7 +1591,7 @@ class iterator : public object { } private: - object value = {}; + object value; }; class type : public object { @@ -1434,7 +1599,9 @@ class type : public object { PYBIND11_OBJECT(type, object, PyType_Check) /// Return a type handle from a handle or an object - static handle handle_of(handle h) { return handle((PyObject *) Py_TYPE(h.ptr())); } + static handle handle_of(handle h) { + return handle(reinterpret_cast(Py_TYPE(h.ptr()))); + } /// Return a type object from a handle or an object static type of(handle h) { return type(type::handle_of(h), borrowed_t{}); } @@ -1512,7 +1679,12 @@ class str : public object { Return a string representation of the object. This is analogous to the ``str()`` function in Python. \endrst */ - explicit str(handle h) : object(raw_str(h.ptr()), stolen_t{}) { + // Templatized to avoid ambiguity with str(const object&) for object-derived types. + template >::value + && std::is_constructible::value, + int> = 0> + explicit str(T &&h) : object(raw_str(handle(std::forward(h)).ptr()), stolen_t{}) { if (!m_ptr) { throw error_already_set(); } @@ -1532,7 +1704,7 @@ class str : public object { if (PyBytes_AsStringAndSize(temp.ptr(), &buffer, &length) != 0) { throw error_already_set(); } - return std::string(buffer, (size_t) length); + return std::string(buffer, static_cast(length)); } template @@ -1553,7 +1725,15 @@ inline namespace literals { /** \rst String literal version of `str` \endrst */ -inline str operator"" _s(const char *s, size_t size) { return {s, size}; } +inline str +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 5 +operator"" _s // gcc 4.8.5 insists on having a space (hard error). +#else +operator""_s // clang 17 generates a deprecation warning if there is a space. +#endif + (const char *s, size_t size) { + return {s, size}; +} } // namespace literals /// \addtogroup pytypes @@ -1724,10 +1904,12 @@ template Unsigned as_unsigned(PyObject *o) { if (sizeof(Unsigned) <= sizeof(unsigned long)) { unsigned long v = PyLong_AsUnsignedLong(o); - return v == (unsigned long) -1 && PyErr_Occurred() ? (Unsigned) -1 : (Unsigned) v; + return v == static_cast(-1) && PyErr_Occurred() ? (Unsigned) -1 + : (Unsigned) v; } unsigned long long v = PyLong_AsUnsignedLongLong(o); - return v == (unsigned long long) -1 && PyErr_Occurred() ? (Unsigned) -1 : (Unsigned) v; + return v == static_cast(-1) && PyErr_Occurred() ? (Unsigned) -1 + : (Unsigned) v; } PYBIND11_NAMESPACE_END(detail) @@ -1771,21 +1953,21 @@ class float_ : public object { PYBIND11_OBJECT_CVT(float_, object, PyFloat_Check, PyNumber_Float) // Allow implicit conversion from float/double: // NOLINTNEXTLINE(google-explicit-constructor) - float_(float value) : object(PyFloat_FromDouble((double) value), stolen_t{}) { + float_(float value) : object(PyFloat_FromDouble(static_cast(value)), stolen_t{}) { if (!m_ptr) { pybind11_fail("Could not allocate float object!"); } } // NOLINTNEXTLINE(google-explicit-constructor) - float_(double value = .0) : object(PyFloat_FromDouble((double) value), stolen_t{}) { + float_(double value = .0) : object(PyFloat_FromDouble(value), stolen_t{}) { if (!m_ptr) { pybind11_fail("Could not allocate float object!"); } } // NOLINTNEXTLINE(google-explicit-constructor) - operator float() const { return (float) PyFloat_AsDouble(m_ptr); } + operator float() const { return static_cast(PyFloat_AsDouble(m_ptr)); } // NOLINTNEXTLINE(google-explicit-constructor) - operator double() const { return (double) PyFloat_AsDouble(m_ptr); } + operator double() const { return PyFloat_AsDouble(m_ptr); } }; class weakref : public object { @@ -1807,13 +1989,14 @@ class weakref : public object { class slice : public object { public: - PYBIND11_OBJECT_DEFAULT(slice, object, PySlice_Check) + PYBIND11_OBJECT(slice, object, PySlice_Check) slice(handle start, handle stop, handle step) : object(PySlice_New(start.ptr(), stop.ptr(), step.ptr()), stolen_t{}) { if (!m_ptr) { pybind11_fail("Could not allocate slice object!"); } } + slice() : slice(none(), none(), none()) {} #ifdef PYBIND11_HAS_OPTIONAL slice(std::optional start, std::optional stop, std::optional step) @@ -1871,28 +2054,13 @@ class capsule : public object { } } + /// Capsule name is nullptr. capsule(const void *value, void (*destructor)(void *)) { - m_ptr = PyCapsule_New(const_cast(value), nullptr, [](PyObject *o) { - // guard if destructor called while err indicator is set - error_scope error_guard; - auto destructor = reinterpret_cast(PyCapsule_GetContext(o)); - if (destructor == nullptr && PyErr_Occurred()) { - throw error_already_set(); - } - const char *name = get_name_in_error_scope(o); - void *ptr = PyCapsule_GetPointer(o, name); - if (ptr == nullptr) { - throw error_already_set(); - } - - if (destructor != nullptr) { - destructor(ptr); - } - }); + initialize_with_void_ptr_destructor(value, nullptr, destructor); + } - if (!m_ptr || PyCapsule_SetContext(m_ptr, reinterpret_cast(destructor)) != 0) { - throw error_already_set(); - } + capsule(const void *value, const char *name, void (*destructor)(void *)) { + initialize_with_void_ptr_destructor(value, name, destructor); } explicit capsule(void (*destructor)()) { @@ -1960,6 +2128,32 @@ class capsule : public object { return name; } + + void initialize_with_void_ptr_destructor(const void *value, + const char *name, + void (*destructor)(void *)) { + m_ptr = PyCapsule_New(const_cast(value), name, [](PyObject *o) { + // guard if destructor called while err indicator is set + error_scope error_guard; + auto destructor = reinterpret_cast(PyCapsule_GetContext(o)); + if (destructor == nullptr && PyErr_Occurred()) { + throw error_already_set(); + } + const char *name = get_name_in_error_scope(o); + void *ptr = PyCapsule_GetPointer(o, name); + if (ptr == nullptr) { + throw error_already_set(); + } + + if (destructor != nullptr) { + destructor(ptr); + } + }); + + if (!m_ptr || PyCapsule_SetContext(m_ptr, reinterpret_cast(destructor)) != 0) { + throw error_already_set(); + } + } }; class tuple : public object { @@ -1973,7 +2167,7 @@ class tuple : public object { pybind11_fail("Could not allocate tuple object!"); } } - size_t size() const { return (size_t) PyTuple_Size(m_ptr); } + size_t size() const { return static_cast(PyTuple_Size(m_ptr)); } bool empty() const { return size() == 0; } detail::tuple_accessor operator[](size_t index) const { return {*this, index}; } template ::value, int> = 0> @@ -2007,7 +2201,7 @@ class dict : public object { typename collector = detail::deferred_t, Args...>> explicit dict(Args &&...args) : dict(collector(std::forward(args)...).kwargs()) {} - size_t size() const { return (size_t) PyDict_Size(m_ptr); } + size_t size() const { return static_cast(PyDict_Size(m_ptr)); } bool empty() const { return size() == 0; } detail::dict_iterator begin() const { return {*this, 0}; } detail::dict_iterator end() const { return {}; } @@ -2027,7 +2221,8 @@ class dict : public object { if (PyDict_Check(op)) { return handle(op).inc_ref().ptr(); } - return PyObject_CallFunctionObjArgs((PyObject *) &PyDict_Type, op, nullptr); + return PyObject_CallFunctionObjArgs( + reinterpret_cast(&PyDict_Type), op, nullptr); } }; @@ -2039,7 +2234,7 @@ class sequence : public object { if (result == -1) { throw error_already_set(); } - return (size_t) result; + return static_cast(result); } bool empty() const { return size() == 0; } detail::sequence_accessor operator[](size_t index) const { return {*this, index}; } @@ -2062,7 +2257,7 @@ class list : public object { pybind11_fail("Could not allocate list object!"); } } - size_t size() const { return (size_t) PyList_Size(m_ptr); } + size_t size() const { return static_cast(PyList_Size(m_ptr)); } bool empty() const { return size() == 0; } detail::list_accessor operator[](size_t index) const { return {*this, index}; } template ::value, int> = 0> @@ -2088,6 +2283,11 @@ class list : public object { throw error_already_set(); } } + void clear() /* py-non-const */ { + if (PyList_SetSlice(m_ptr, 0, PyList_Size(m_ptr), nullptr) == -1) { + throw error_already_set(); + } + } }; class args : public tuple { @@ -2097,6 +2297,18 @@ class kwargs : public dict { PYBIND11_OBJECT_DEFAULT(kwargs, dict, PyDict_Check) }; +// Subclasses of args and kwargs to support type hinting +// as defined in PEP 484. See #5357 for more info. +template +class Args : public args { + using args::args; +}; + +template +class KWArgs : public kwargs { + using kwargs::kwargs; +}; + class anyset : public object { public: PYBIND11_OBJECT(anyset, object, PyAnySet_Check) @@ -2331,7 +2543,7 @@ inline size_t len(handle h) { if (result < 0) { throw error_already_set(); } - return (size_t) result; + return static_cast(result); } /// Get the length hint of a Python object. @@ -2344,7 +2556,7 @@ inline size_t len_hint(handle h) { PyErr_Clear(); return 0; } - return (size_t) result; + return static_cast(result); } inline str repr(handle h) { @@ -2417,6 +2629,20 @@ str_attr_accessor object_api::doc() const { return attr("__doc__"); } +template +object object_api::annotations() const { +// This is needed again because of the lazy annotations added in 3.14+ +#if PY_VERSION_HEX < 0x030A0000 || PY_VERSION_HEX >= 0x030E0000 + // https://docs.python.org/3/howto/annotations.html#accessing-the-annotations-dict-of-an-object-in-python-3-9-and-older + if (!hasattr(derived(), "__annotations__")) { + setattr(derived(), "__annotations__", dict()); + } + return attr("__annotations__"); +#else + return getattr(derived(), "__annotations__", dict()); +#endif +} + template handle object_api::get_type() const { return type::handle_of(derived()); @@ -2483,5 +2709,18 @@ PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator>>=, PyNumber_InPlaceRshift) #undef PYBIND11_MATH_OPERATOR_BINARY #undef PYBIND11_MATH_OPERATOR_BINARY_INPLACE +// Meant to return a Python str, but this is not checked. +inline object get_module_name_if_available(handle scope) { + if (scope) { + if (hasattr(scope, "__module__")) { + return scope.attr("__module__"); + } + if (hasattr(scope, "__name__")) { + return scope.attr("__name__"); + } + } + return object(); +} + PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/stl.h b/lib/pybind11/include/pybind11/stl.h index 84c5171..01be0b4 100644 --- a/lib/pybind11/include/pybind11/stl.h +++ b/lib/pybind11/include/pybind11/stl.h @@ -11,10 +11,14 @@ #include "pybind11.h" #include "detail/common.h" +#include "detail/descr.h" +#include "detail/type_caster_base.h" #include +#include #include #include +#include #include #include #include @@ -35,6 +39,103 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) +// +// Begin: Equivalent of +// https://github.com/google/clif/blob/ae4eee1de07cdf115c0c9bf9fec9ff28efce6f6c/clif/python/runtime.cc#L388-L438 +/* +The three `object_is_convertible_to_*()` functions below are +the result of converging the behaviors of pybind11 and PyCLIF +(http://github.com/google/clif). + +Originally PyCLIF was extremely far on the permissive side of the spectrum, +while pybind11 was very far on the strict side. Originally PyCLIF accepted any +Python iterable as input for a C++ `vector`/`set`/`map` argument, as long as +the elements were convertible. The obvious (in hindsight) problem was that +any empty Python iterable could be passed to any of these C++ types, e.g. `{}` +was accepted for C++ `vector`/`set` arguments, or `[]` for C++ `map` arguments. + +The functions below strike a practical permissive-vs-strict compromise, +informed by tens of thousands of use cases in the wild. A main objective is +to prevent accidents and improve readability: + +- Python literals must match the C++ types. + +- For C++ `set`: The potentially reducing conversion from a Python sequence + (e.g. Python `list` or `tuple`) to a C++ `set` must be explicit, by going + through a Python `set`. + +- However, a Python `set` can still be passed to a C++ `vector`. The rationale + is that this conversion is not reducing. Implicit conversions of this kind + are also fairly commonly used, therefore enforcing explicit conversions + would have an unfavorable cost : benefit ratio; more sloppily speaking, + such an enforcement would be more annoying than helpful. + +Additional checks have been added to allow types derived from `collections.abc.Set` and +`collections.abc.Mapping` (`collections.abc.Sequence` is already allowed by `PySequence_Check`). +*/ + +inline bool object_is_instance_with_one_of_tp_names(PyObject *obj, + std::initializer_list tp_names) { + if (PyType_Check(obj)) { + return false; + } + const char *obj_tp_name = Py_TYPE(obj)->tp_name; + for (const auto *tp_name : tp_names) { + if (std::strcmp(obj_tp_name, tp_name) == 0) { + return true; + } + } + return false; +} + +inline bool object_is_convertible_to_std_vector(const handle &src) { + // Allow sequence-like objects, but not (byte-)string-like objects. + if (PySequence_Check(src.ptr()) != 0) { + return !PyUnicode_Check(src.ptr()) && !PyBytes_Check(src.ptr()); + } + // Allow generators, set/frozenset and several common iterable types. + return (PyGen_Check(src.ptr()) != 0) || (PyAnySet_Check(src.ptr()) != 0) + || object_is_instance_with_one_of_tp_names( + src.ptr(), {"dict_keys", "dict_values", "dict_items", "map", "zip"}); +} + +inline bool object_is_convertible_to_std_set(const handle &src, bool convert) { + // Allow set/frozenset and dict keys. + // In convert mode: also allow types derived from collections.abc.Set. + return ((PyAnySet_Check(src.ptr()) != 0) + || object_is_instance_with_one_of_tp_names(src.ptr(), {"dict_keys"})) + || (convert && isinstance(src, module_::import("collections.abc").attr("Set"))); +} + +inline bool object_is_convertible_to_std_map(const handle &src, bool convert) { + // Allow dict. + if (PyDict_Check(src.ptr())) { + return true; + } + // Allow types conforming to Mapping Protocol. + // According to https://docs.python.org/3/c-api/mapping.html, `PyMappingCheck()` checks for + // `__getitem__()` without checking the type of keys. In order to restrict the allowed types + // closer to actual Mapping-like types, we also check for the `items()` method. + if (PyMapping_Check(src.ptr()) != 0) { + PyObject *items = PyObject_GetAttrString(src.ptr(), "items"); + if (items != nullptr) { + bool is_convertible = (PyCallable_Check(items) != 0); + Py_DECREF(items); + if (is_convertible) { + return true; + } + } else { + PyErr_Clear(); + } + } + // In convert mode: Allow types derived from collections.abc.Mapping + return convert && isinstance(src, module_::import("collections.abc").attr("Mapping")); +} + +// +// End: Equivalent of clif/python/runtime.cc +// + /// Extracts an const lvalue reference or rvalue reference for U based on the type of T (e.g. for /// forwarding a container element). Typically used indirect via forwarded_type(), below. template @@ -66,17 +167,10 @@ struct set_caster { } void reserve_maybe(const anyset &, void *) {} -public: - bool load(handle src, bool convert) { - if (!isinstance(src)) { - return false; - } - auto s = reinterpret_borrow(src); - value.clear(); - reserve_maybe(s, &value); - for (auto entry : s) { + bool convert_iterable(const iterable &itbl, bool convert) { + for (const auto &it : itbl) { key_conv conv; - if (!conv.load(entry, convert)) { + if (!conv.load(it, convert)) { return false; } value.insert(cast_op(std::move(conv))); @@ -84,6 +178,29 @@ struct set_caster { return true; } + bool convert_anyset(const anyset &s, bool convert) { + value.clear(); + reserve_maybe(s, &value); + return convert_iterable(s, convert); + } + +public: + bool load(handle src, bool convert) { + if (!object_is_convertible_to_std_set(src, convert)) { + return false; + } + if (isinstance(src)) { + value.clear(); + return convert_anyset(reinterpret_borrow(src), convert); + } + if (!convert) { + return false; + } + assert(isinstance(src)); + value.clear(); + return convert_iterable(reinterpret_borrow(src), convert); + } + template static handle cast(T &&src, return_value_policy policy, handle parent) { if (!std::is_lvalue_reference::value) { @@ -100,7 +217,9 @@ struct set_caster { return s.release(); } - PYBIND11_TYPE_CASTER(type, const_name("Set[") + key_conv::name + const_name("]")); + PYBIND11_TYPE_CASTER(type, + io_name("collections.abc.Set", "set") + const_name("[") + key_conv::name + + const_name("]")); }; template @@ -115,15 +234,10 @@ struct map_caster { } void reserve_maybe(const dict &, void *) {} -public: - bool load(handle src, bool convert) { - if (!isinstance(src)) { - return false; - } - auto d = reinterpret_borrow(src); + bool convert_elements(const dict &d, bool convert) { value.clear(); reserve_maybe(d, &value); - for (auto it : d) { + for (const auto &it : d) { key_conv kconv; value_conv vconv; if (!kconv.load(it.first.ptr(), convert) || !vconv.load(it.second.ptr(), convert)) { @@ -134,6 +248,25 @@ struct map_caster { return true; } +public: + bool load(handle src, bool convert) { + if (!object_is_convertible_to_std_map(src, convert)) { + return false; + } + if (isinstance(src)) { + return convert_elements(reinterpret_borrow(src), convert); + } + if (!convert) { + return false; + } + auto items = reinterpret_steal(PyMapping_Items(src.ptr())); + if (!items) { + throw error_already_set(); + } + assert(isinstance(items)); + return convert_elements(dict(reinterpret_borrow(items)), convert); + } + template static handle cast(T &&src, return_value_policy policy, handle parent) { dict d; @@ -157,7 +290,8 @@ struct map_caster { } PYBIND11_TYPE_CASTER(Type, - const_name("Dict[") + key_conv::name + const_name(", ") + value_conv::name + io_name("collections.abc.Mapping", "dict") + const_name("[") + + key_conv::name + const_name(", ") + value_conv::name + const_name("]")); }; @@ -166,13 +300,35 @@ struct list_caster { using value_conv = make_caster; bool load(handle src, bool convert) { - if (!isinstance(src) || isinstance(src) || isinstance(src)) { + if (!object_is_convertible_to_std_vector(src)) { return false; } - auto s = reinterpret_borrow(src); + if (isinstance(src)) { + return convert_elements(src, convert); + } + if (!convert) { + return false; + } + // Designed to be behavior-equivalent to passing tuple(src) from Python: + // The conversion to a tuple will first exhaust the generator object, to ensure that + // the generator is not left in an unpredictable (to the caller) partially-consumed + // state. + assert(isinstance(src)); + return convert_elements(tuple(reinterpret_borrow(src)), convert); + } + +private: + template ::value, int> = 0> + void reserve_maybe(const sequence &s, Type *) { + value.reserve(s.size()); + } + void reserve_maybe(const sequence &, void *) {} + + bool convert_elements(handle seq, bool convert) { + auto s = reinterpret_borrow(seq); value.clear(); reserve_maybe(s, &value); - for (auto it : s) { + for (const auto &it : seq) { value_conv conv; if (!conv.load(it, convert)) { return false; @@ -182,13 +338,6 @@ struct list_caster { return true; } -private: - template ::value, int> = 0> - void reserve_maybe(const sequence &s, Type *) { - value.reserve(s.size()); - } - void reserve_maybe(const sequence &, void *) {} - public: template static handle cast(T &&src, return_value_policy policy, handle parent) { @@ -208,7 +357,9 @@ struct list_caster { return l.release(); } - PYBIND11_TYPE_CASTER(Type, const_name("List[") + value_conv::name + const_name("]")); + PYBIND11_TYPE_CASTER(Type, + io_name("collections.abc.Sequence", "list") + const_name("[") + + value_conv::name + const_name("]")); }; template @@ -220,43 +371,87 @@ struct type_caster> : list_caster struct type_caster> : list_caster, Type> {}; +template +ArrayType vector_to_array_impl(V &&v, index_sequence) { + return {{std::move(v[I])...}}; +} + +// Based on https://en.cppreference.com/w/cpp/container/array/to_array +template +ArrayType vector_to_array(V &&v) { + return vector_to_array_impl(std::forward(v), make_index_sequence{}); +} + template struct array_caster { using value_conv = make_caster; private: - template - bool require_size(enable_if_t size) { - if (value.size() != size) { - value.resize(size); + std::unique_ptr value; + + template = 0> + bool convert_elements(handle seq, bool convert) { + auto l = reinterpret_borrow(seq); + value.reset(new ArrayType{}); + // Using `resize` to preserve the behavior exactly as it was before PR #5305 + // For the `resize` to work, `Value` must be default constructible. + // For `std::valarray`, this is a requirement: + // https://en.cppreference.com/w/cpp/named_req/NumericType + value->resize(l.size()); + size_t ctr = 0; + for (const auto &it : l) { + value_conv conv; + if (!conv.load(it, convert)) { + return false; + } + (*value)[ctr++] = cast_op(std::move(conv)); } return true; } - template - bool require_size(enable_if_t size) { - return size == Size; - } -public: - bool load(handle src, bool convert) { - if (!isinstance(src)) { - return false; - } - auto l = reinterpret_borrow(src); - if (!require_size(l.size())) { + template = 0> + bool convert_elements(handle seq, bool convert) { + auto l = reinterpret_borrow(seq); + if (l.size() != Size) { return false; } - size_t ctr = 0; + // The `temp` storage is needed to support `Value` types that are not + // default-constructible. + // Deliberate choice: no template specializations, for simplicity, and + // because the compile time overhead for the specializations is deemed + // more significant than the runtime overhead for the `temp` storage. + std::vector temp; + temp.reserve(l.size()); for (auto it : l) { value_conv conv; if (!conv.load(it, convert)) { return false; } - value[ctr++] = cast_op(std::move(conv)); + temp.emplace_back(cast_op(std::move(conv))); } + value.reset(new ArrayType(vector_to_array(std::move(temp)))); return true; } +public: + bool load(handle src, bool convert) { + if (!object_is_convertible_to_std_vector(src)) { + return false; + } + if (isinstance(src)) { + return convert_elements(src, convert); + } + if (!convert) { + return false; + } + // Designed to be behavior-equivalent to passing tuple(src) from Python: + // The conversion to a tuple will first exhaust the generator object, to ensure that + // the generator is not left in an unpredictable (to the caller) partially-consumed + // state. + assert(isinstance(src)); + return convert_elements(tuple(reinterpret_borrow(src)), convert); + } + template static handle cast(T &&src, return_value_policy policy, handle parent) { list l(src.size()); @@ -272,12 +467,38 @@ struct array_caster { return l.release(); } - PYBIND11_TYPE_CASTER(ArrayType, - const_name("List[") + value_conv::name - + const_name(const_name(""), - const_name("[") + const_name() - + const_name("]")) - + const_name("]")); + // Code copied from PYBIND11_TYPE_CASTER macro. + // Intentionally preserving the behavior exactly as it was before PR #5305 + template >::value, int> = 0> + static handle cast(T_ *src, return_value_policy policy, handle parent) { + if (!src) { + return none().release(); + } + if (policy == return_value_policy::take_ownership) { + auto h = cast(std::move(*src), policy, parent); + delete src; // WARNING: Assumes `src` was allocated with `new`. + return h; + } + return cast(*src, policy, parent); + } + + // NOLINTNEXTLINE(google-explicit-constructor) + operator ArrayType *() { return &(*value); } + // NOLINTNEXTLINE(google-explicit-constructor) + operator ArrayType &() { return *value; } + // NOLINTNEXTLINE(google-explicit-constructor) + operator ArrayType &&() && { return std::move(*value); } + + template + using cast_op_type = movable_cast_op_type; + + static constexpr auto name + = const_name(const_name(""), const_name("typing.Annotated[")) + + io_name("collections.abc.Sequence", "list") + const_name("[") + value_conv::name + + const_name("]") + + const_name(const_name(""), + const_name(", \"FixedSize(") + const_name() + + const_name(")\"]")); }; template @@ -336,7 +557,7 @@ struct optional_caster { return true; } - PYBIND11_TYPE_CASTER(Type, const_name("Optional[") + value_conv::name + const_name("]")); + PYBIND11_TYPE_CASTER(Type, value_conv::name | make_caster::name); }; #if defined(PYBIND11_HAS_OPTIONAL) @@ -420,9 +641,7 @@ struct variant_caster> { } using Type = V; - PYBIND11_TYPE_CASTER(Type, - const_name("Union[") + detail::concat(make_caster::name...) - + const_name("]")); + PYBIND11_TYPE_CASTER(Type, ::pybind11::detail::union_concat(make_caster::name...)); }; #if defined(PYBIND11_HAS_VARIANT) diff --git a/lib/pybind11/include/pybind11/stl/filesystem.h b/lib/pybind11/include/pybind11/stl/filesystem.h index e26f421..52d2962 100644 --- a/lib/pybind11/include/pybind11/stl/filesystem.h +++ b/lib/pybind11/include/pybind11/stl/filesystem.h @@ -4,36 +4,32 @@ #pragma once -#include "../pybind11.h" -#include "../detail/common.h" -#include "../detail/descr.h" -#include "../cast.h" -#include "../pytypes.h" +#include +#include +#include +#include +#include #include -#ifdef __has_include -# if defined(PYBIND11_CPP17) -# if __has_include() && \ - PY_VERSION_HEX >= 0x03060000 -# include -# define PYBIND11_HAS_FILESYSTEM 1 -# elif __has_include() -# include -# define PYBIND11_HAS_EXPERIMENTAL_FILESYSTEM 1 -# endif -# endif -#endif - -#if !defined(PYBIND11_HAS_FILESYSTEM) && !defined(PYBIND11_HAS_EXPERIMENTAL_FILESYSTEM) \ - && !defined(PYBIND11_HAS_FILESYSTEM_IS_OPTIONAL) -# error \ - "Neither #include nor #include +#elif defined(PYBIND11_HAS_EXPERIMENTAL_FILESYSTEM) +# include +#else +# error "Neither #include nor #include is available." #endif PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) +#ifdef PYPY_VERSION +# define PYBIND11_REINTERPRET_CAST_VOID_PTR_IF_NOT_PYPY(...) (__VA_ARGS__) +#else +# define PYBIND11_REINTERPRET_CAST_VOID_PTR_IF_NOT_PYPY(...) \ + (reinterpret_cast(__VA_ARGS__)) +#endif + #if defined(PYBIND11_HAS_FILESYSTEM) || defined(PYBIND11_HAS_EXPERIMENTAL_FILESYSTEM) template struct path_caster { @@ -73,7 +69,8 @@ struct path_caster { } PyObject *native = nullptr; if constexpr (std::is_same_v) { - if (PyUnicode_FSConverter(buf, &native) != 0) { + if (PyUnicode_FSConverter(buf, PYBIND11_REINTERPRET_CAST_VOID_PTR_IF_NOT_PYPY(&native)) + != 0) { if (auto *c_str = PyBytes_AsString(native)) { // AsString returns a pointer to the internal buffer, which // must not be free'd. @@ -81,7 +78,8 @@ struct path_caster { } } } else if constexpr (std::is_same_v) { - if (PyUnicode_FSDecoder(buf, &native) != 0) { + if (PyUnicode_FSDecoder(buf, PYBIND11_REINTERPRET_CAST_VOID_PTR_IF_NOT_PYPY(&native)) + != 0) { if (auto *c_str = PyUnicode_AsWideCharString(native, nullptr)) { // AsWideCharString returns a new string that must be free'd. value = c_str; // Copies the string. @@ -98,7 +96,7 @@ struct path_caster { return true; } - PYBIND11_TYPE_CASTER(T, const_name("os.PathLike")); + PYBIND11_TYPE_CASTER(T, io_name("os.PathLike | str | bytes", "pathlib.Path")); }; #endif // PYBIND11_HAS_FILESYSTEM || defined(PYBIND11_HAS_EXPERIMENTAL_FILESYSTEM) diff --git a/lib/pybind11/include/pybind11/stl_bind.h b/lib/pybind11/include/pybind11/stl_bind.h index 7cfd996..360c2cd 100644 --- a/lib/pybind11/include/pybind11/stl_bind.h +++ b/lib/pybind11/include/pybind11/stl_bind.h @@ -61,9 +61,11 @@ struct is_comparable< /* For a vector/map data structure, recursively check the value type (which is std::pair for maps) */ template -struct is_comparable::is_vector>> { - static constexpr const bool value = is_comparable::value; -}; +struct is_comparable::is_vector>> + : is_comparable::type_to_check_recursively> {}; + +template <> +struct is_comparable : std::true_type {}; /* For pairs, recursively check the two data types */ template @@ -156,8 +158,7 @@ void vector_modifiers( return v.release(); })); - cl.def( - "clear", [](Vector &v) { v.clear(); }, "Clear the contents"); + cl.def("clear", [](Vector &v) { v.clear(); }, "Clear the contents"); cl.def( "extend", @@ -179,7 +180,7 @@ void vector_modifiers( v.end()); try { v.shrink_to_fit(); - } catch (const std::exception &) { + } catch (const std::exception &) { // NOLINT(bugprone-empty-catch) // Do nothing } throw; @@ -243,7 +244,7 @@ void vector_modifiers( } auto *seq = new Vector(); - seq->reserve((size_t) slicelength); + seq->reserve(slicelength); for (size_t i = 0; i < slicelength; ++i) { seq->push_back(v[start]); @@ -285,18 +286,28 @@ void vector_modifiers( cl.def( "__delitem__", [](Vector &v, const slice &slice) { - size_t start = 0, stop = 0, step = 0, slicelength = 0; + ssize_t start = 0, stop = 0, step = 0, slicelength = 0; - if (!slice.compute(v.size(), &start, &stop, &step, &slicelength)) { + if (!slice.compute( + static_cast(v.size()), &start, &stop, &step, &slicelength)) { throw error_already_set(); } - if (step == 1 && false) { + if (step == 1) { v.erase(v.begin() + (DiffType) start, v.begin() + DiffType(start + slicelength)); - } else { - for (size_t i = 0; i < slicelength; ++i) { + } else if (slicelength > 0) { + // Erase non-contiguous slices in descending index order so that + // erasing an element never shifts an index that remains to be erased. + if (step > 0) { + start += (slicelength - 1) * step; + step = -step; + } + while (true) { v.erase(v.begin() + DiffType(start)); - start += step - 1; + if (--slicelength == 0) { + break; + } + start += step; } } }, @@ -486,7 +497,7 @@ PYBIND11_NAMESPACE_END(detail) // // std::vector // -template , typename... Args> +template , typename... Args> class_ bind_vector(handle scope, std::string const &name, Args &&...args) { using Class_ = class_; @@ -523,7 +534,7 @@ class_ bind_vector(handle scope, std::string const &name, A [](const Vector &v) -> bool { return !v.empty(); }, "Check whether the list is nonempty"); - cl.def("__len__", &Vector::size); + cl.def("__len__", [](const Vector &vec) { return vec.size(); }); #if 0 // C++ style functions deprecated, leaving it here as an example @@ -643,66 +654,99 @@ auto map_if_insertion_operator(Class_ &cl, std::string const &name) "Return the canonical string representation of this map."); } -template struct keys_view { virtual size_t len() = 0; virtual iterator iter() = 0; - virtual bool contains(const KeyType &k) = 0; - virtual bool contains(const object &k) = 0; + virtual bool contains(const handle &k) = 0; virtual ~keys_view() = default; }; -template struct values_view { virtual size_t len() = 0; virtual iterator iter() = 0; virtual ~values_view() = default; }; -template struct items_view { virtual size_t len() = 0; virtual iterator iter() = 0; virtual ~items_view() = default; }; -template -struct KeysViewImpl : public KeysView { +template +struct KeysViewImpl : public detail::keys_view { explicit KeysViewImpl(Map &map) : map(map) {} size_t len() override { return map.size(); } iterator iter() override { return make_key_iterator(map.begin(), map.end()); } - bool contains(const typename Map::key_type &k) override { return map.find(k) != map.end(); } - bool contains(const object &) override { return false; } + bool contains(const handle &k) override { + try { + return map.find(k.template cast()) != map.end(); + } catch (const cast_error &) { + return false; + } + } Map ↦ }; -template -struct ValuesViewImpl : public ValuesView { +template +struct ValuesViewImpl : public detail::values_view { explicit ValuesViewImpl(Map &map) : map(map) {} size_t len() override { return map.size(); } iterator iter() override { return make_value_iterator(map.begin(), map.end()); } Map ↦ }; -template -struct ItemsViewImpl : public ItemsView { +template +struct ItemsViewImpl : public detail::items_view { explicit ItemsViewImpl(Map &map) : map(map) {} size_t len() override { return map.size(); } iterator iter() override { return make_iterator(map.begin(), map.end()); } Map ↦ }; +inline str format_message_key_error_key_object(handle py_key) { + str message = "pybind11::bind_map key"; + if (!py_key) { + return message; + } + try { + message = str(py_key); + } catch (const std::exception &) { + try { + message = repr(py_key); + } catch (const std::exception &) { + return message; + } + } + const ssize_t cut_length = 100; + if (len(message) > 2 * cut_length + 3) { + return str(message[slice(0, cut_length, 1)]) + str("✄✄✄") + + str(message[slice(-cut_length, static_cast(len(message)), 1)]); + } + return message; +} + +template +str format_message_key_error(const KeyType &key) { + object py_key; + try { + py_key = cast(key); + } catch (const std::exception &) { + do { // Trick to avoid "empty catch" warning/error. + } while (false); + } + return format_message_key_error_key_object(py_key); +} + PYBIND11_NAMESPACE_END(detail) -template , typename... Args> +template , typename... Args> class_ bind_map(handle scope, const std::string &name, Args &&...args) { using KeyType = typename Map::key_type; using MappedType = typename Map::mapped_type; - using StrippedKeyType = detail::remove_cvref_t; - using StrippedMappedType = detail::remove_cvref_t; - using KeysView = detail::keys_view; - using ValuesView = detail::values_view; - using ItemsView = detail::items_view; + using KeysView = detail::keys_view; + using ValuesView = detail::values_view; + using ItemsView = detail::items_view; using Class_ = class_; // If either type is a non-module-local bound type then make the map binding non-local as well; @@ -716,39 +760,20 @@ class_ bind_map(handle scope, const std::string &name, Args && } Class_ cl(scope, name.c_str(), pybind11::module_local(local), std::forward(args)...); - static constexpr auto key_type_descr = detail::make_caster::name; - static constexpr auto mapped_type_descr = detail::make_caster::name; - std::string key_type_name(key_type_descr.text), mapped_type_name(mapped_type_descr.text); - // If key type isn't properly wrapped, fall back to C++ names - if (key_type_name == "%") { - key_type_name = detail::type_info_description(typeid(KeyType)); - } - // Similarly for value type: - if (mapped_type_name == "%") { - mapped_type_name = detail::type_info_description(typeid(MappedType)); - } - - // Wrap KeysView[KeyType] if it wasn't already wrapped + // Wrap KeysView if it wasn't already wrapped if (!detail::get_type_info(typeid(KeysView))) { - class_ keys_view( - scope, ("KeysView[" + key_type_name + "]").c_str(), pybind11::module_local(local)); + class_ keys_view(scope, "KeysView", pybind11::module_local(local)); keys_view.def("__len__", &KeysView::len); keys_view.def("__iter__", &KeysView::iter, keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */ ); - keys_view.def("__contains__", - static_cast(&KeysView::contains)); - // Fallback for when the object is not of the key type - keys_view.def("__contains__", - static_cast(&KeysView::contains)); + keys_view.def("__contains__", &KeysView::contains); } // Similarly for ValuesView: if (!detail::get_type_info(typeid(ValuesView))) { - class_ values_view(scope, - ("ValuesView[" + mapped_type_name + "]").c_str(), - pybind11::module_local(local)); + class_ values_view(scope, "ValuesView", pybind11::module_local(local)); values_view.def("__len__", &ValuesView::len); values_view.def("__iter__", &ValuesView::iter, @@ -757,10 +782,7 @@ class_ bind_map(handle scope, const std::string &name, Args && } // Similarly for ItemsView: if (!detail::get_type_info(typeid(ItemsView))) { - class_ items_view( - scope, - ("ItemsView[" + key_type_name + ", ").append(mapped_type_name + "]").c_str(), - pybind11::module_local(local)); + class_ items_view(scope, "ItemsView", pybind11::module_local(local)); items_view.def("__len__", &ItemsView::len); items_view.def("__iter__", &ItemsView::iter, @@ -786,25 +808,19 @@ class_ bind_map(handle scope, const std::string &name, Args && cl.def( "keys", - [](Map &m) { - return std::unique_ptr(new detail::KeysViewImpl(m)); - }, + [](Map &m) { return std::unique_ptr(new detail::KeysViewImpl(m)); }, keep_alive<0, 1>() /* Essential: keep map alive while view exists */ ); cl.def( "values", - [](Map &m) { - return std::unique_ptr(new detail::ValuesViewImpl(m)); - }, + [](Map &m) { return std::unique_ptr(new detail::ValuesViewImpl(m)); }, keep_alive<0, 1>() /* Essential: keep map alive while view exists */ ); cl.def( "items", - [](Map &m) { - return std::unique_ptr(new detail::ItemsViewImpl(m)); - }, + [](Map &m) { return std::unique_ptr(new detail::ItemsViewImpl(m)); }, keep_alive<0, 1>() /* Essential: keep map alive while view exists */ ); @@ -813,7 +829,8 @@ class_ bind_map(handle scope, const std::string &name, Args && [](Map &m, const KeyType &k) -> MappedType & { auto it = m.find(k); if (it == m.end()) { - throw key_error(); + set_error(PyExc_KeyError, detail::format_message_key_error(k)); + throw error_already_set(); } return it->second; }, @@ -836,12 +853,14 @@ class_ bind_map(handle scope, const std::string &name, Args && cl.def("__delitem__", [](Map &m, const KeyType &k) { auto it = m.find(k); if (it == m.end()) { - throw key_error(); + set_error(PyExc_KeyError, detail::format_message_key_error(k)); + throw error_already_set(); } m.erase(it); }); - cl.def("__len__", &Map::size); + // Always use a lambda in case of `using` declaration + cl.def("__len__", [](const Map &m) { return m.size(); }); return cl; } diff --git a/lib/pybind11/include/pybind11/subinterpreter.h b/lib/pybind11/include/pybind11/subinterpreter.h new file mode 100644 index 0000000..f1e4e44 --- /dev/null +++ b/lib/pybind11/include/pybind11/subinterpreter.h @@ -0,0 +1,465 @@ +/* + pybind11/subinterpreter.h: Support for creating and using subinterpreters + + Copyright (c) 2025 The Pybind Development Team. + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "detail/common.h" +#include "detail/internals.h" +#include "gil.h" + +#include + +#ifndef PYBIND11_HAS_SUBINTERPRETER_SUPPORT +# error "This platform does not support subinterpreters, do not include this file." +#endif + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +class subinterpreter; +class subinterpreter_thread_state; + +/// Activate the subinterpreter and acquire its GIL, while also releasing any GIL and interpreter +/// currently held. Upon exiting the scope, the previous subinterpreter (if any) and its +/// associated GIL are restored to their state as they were before the scope was entered. +/// +/// Two construction modes are supported: +/// +/// 1. `subinterpreter_scoped_activate(subinterpreter const &)`: +/// Transient mode (the default). A fresh PyThreadState is created on entry and destroyed on +/// exit. This is the established behavior; existing code is unaffected. +/// +/// 2. `subinterpreter_scoped_activate(subinterpreter_thread_state &)`: +/// Reuse mode. The PyThreadState owned by the given subinterpreter_thread_state is swapped +/// in on entry and swapped out (but NOT destroyed) on exit, so repeated activations on the +/// same OS thread reuse the same PyThreadState and preserve its per-thread interpreter state. +/// Use this when a single OS thread re-enters one or more subinterpreters many times. +class subinterpreter_scoped_activate { +public: + explicit subinterpreter_scoped_activate(subinterpreter const &si); + explicit subinterpreter_scoped_activate(subinterpreter_thread_state &ts); + ~subinterpreter_scoped_activate(); + + subinterpreter_scoped_activate(subinterpreter_scoped_activate &&) = delete; + subinterpreter_scoped_activate(subinterpreter_scoped_activate const &) = delete; + subinterpreter_scoped_activate &operator=(subinterpreter_scoped_activate &) = delete; + subinterpreter_scoped_activate &operator=(subinterpreter_scoped_activate const &) = delete; + +private: + PyThreadState *old_tstate_ = nullptr; + PyThreadState *tstate_ = nullptr; + PyGILState_STATE gil_state_; + bool simple_gil_ = false; + // When true, tstate_ is owned by a subinterpreter_thread_state and must NOT be destroyed + // when this scope exits (only swapped out). + bool borrowed_ = false; +}; + +/// Holds a Python subinterpreter instance +class subinterpreter { +public: + /// empty/unusable, but move-assignable. use create() to create a subinterpreter. + subinterpreter() = default; + + subinterpreter(subinterpreter const ©) = delete; + subinterpreter &operator=(subinterpreter const ©) = delete; + + subinterpreter(subinterpreter &&old) noexcept + : istate_(old.istate_), creation_tstate_(old.creation_tstate_) { + old.istate_ = nullptr; + old.creation_tstate_ = nullptr; + } + + subinterpreter &operator=(subinterpreter &&old) noexcept { + std::swap(old.istate_, istate_); + std::swap(old.creation_tstate_, creation_tstate_); + return *this; + } + + /// Create a new subinterpreter with the specified configuration + /// @note This function acquires (and then releases) the main interpreter GIL, but the main + /// interpreter and its GIL are not required to be held prior to calling this function. + static subinterpreter create(PyInterpreterConfig const &cfg) { + + subinterpreter result; + { + // we must hold the main GIL in order to create a subinterpreter + subinterpreter_scoped_activate main_guard(main()); + + // error_scope reads and writes the *current* thread state, so it must be constructed + // only after main_guard has attached one (see PR #6127 for details). + error_scope err_scope; + + auto *prev_tstate = PyThreadState_Get(); + + PyStatus status; + + { + /* + Several internal CPython modules are lacking proper subinterpreter support in 3.12 + even though it is "stable" in that version. This most commonly seems to cause + crashes when two interpreters concurrently initialize, which imports several things + (like builtins, unicode, codecs). + */ +#if PY_VERSION_HEX < 0x030D0000 && defined(Py_MOD_PER_INTERPRETER_GIL_SUPPORTED) + static std::mutex one_at_a_time; + std::lock_guard guard(one_at_a_time); +#endif + status = Py_NewInterpreterFromConfig(&result.creation_tstate_, &cfg); + } + + // this doesn't raise a normal Python exception, it provides an exit() status code. + if (PyStatus_Exception(status) != 0) { + pybind11_fail("failed to create new sub-interpreter"); + } + + // upon success, the new interpreter is activated in this thread + result.istate_ = result.creation_tstate_->interp; + detail::has_seen_non_main_interpreter() = true; + detail::get_internals(); // initialize internals.tstate, amongst other things... + + // In 3.13+ this state should be deleted right away, and the memory will be reused for + // the next threadstate on this interpreter. However, on 3.12 we cannot do that, we + // must keep it around (but not use it) ... see destructor. +#if PY_VERSION_HEX >= 0x030D0000 + PyThreadState_Clear(result.creation_tstate_); + PyThreadState_DeleteCurrent(); +#endif + + // we have to switch back to main, and then the scopes will handle cleanup + PyThreadState_Swap(prev_tstate); + } + return result; + } + + /// Calls create() with a default configuration of an isolated interpreter that disallows fork, + /// exec, and Python threads. + static subinterpreter create() { + // same as the default config in the python docs + PyInterpreterConfig cfg; + std::memset(&cfg, 0, sizeof(cfg)); + cfg.allow_threads = 1; + cfg.check_multi_interp_extensions = 1; + cfg.gil = PyInterpreterConfig_OWN_GIL; + return create(cfg); + } + + ~subinterpreter() { + if (!creation_tstate_) { + // non-owning wrapper, do nothing. + return; + } + + PyThreadState *destroy_tstate = nullptr; + PyThreadState *old_tstate = nullptr; + + // Python 3.12 requires us to keep the original PyThreadState alive until we are ready to + // destroy the interpreter. We prefer to use that to destroy the interpreter. +#if PY_VERSION_HEX < 0x030D0000 + // The tstate passed to Py_EndInterpreter MUST have been created on the current OS thread. + bool same_thread = false; +# ifdef PY_HAVE_THREAD_NATIVE_ID + same_thread = PyThread_get_thread_native_id() == creation_tstate_->native_thread_id; +# endif + if (same_thread) { + // OK it is safe to use the creation state here + destroy_tstate = creation_tstate_; + old_tstate = PyThreadState_Swap(destroy_tstate); + } else { + // We have to make a new tstate on this thread and use that. + destroy_tstate = PyThreadState_New(istate_); + old_tstate = PyThreadState_Swap(destroy_tstate); + + // We can use the one we just created, so we must delete the creation state. + PyThreadState_Clear(creation_tstate_); + PyThreadState_Delete(creation_tstate_); + } +#else + destroy_tstate = PyThreadState_New(istate_); + old_tstate = PyThreadState_Swap(destroy_tstate); +#endif + + bool switch_back = (old_tstate != nullptr) && old_tstate->interp != istate_; + + // Internals always exists in the subinterpreter, this class enforces it when it creates + // the subinterpreter. Even if it didn't, this only creates the pointer-to-pointer, not the + // internals themselves. + detail::get_internals_pp_manager().get_pp(); + detail::get_local_internals_pp_manager().get_pp(); + + // End it + Py_EndInterpreter(destroy_tstate); + + // It's possible for the internals to be created during endinterpreter (e.g. if a + // py::capsule calls `get_internals()` during destruction), so we destroy afterward. + detail::get_internals_pp_manager().destroy(); + detail::get_local_internals_pp_manager().destroy(); + + // switch back to the old tstate and old GIL (if there was one) + if (switch_back) { + PyThreadState_Swap(old_tstate); + } + } + + /// Get a handle to the main interpreter that can be used with subinterpreter_scoped_activate + /// Note that destructing the handle is a noop, the main interpreter can only be ended by + /// py::finalize_interpreter() + static subinterpreter main() { + subinterpreter m; + m.istate_ = PyInterpreterState_Main(); + m.disarm(); // make destruct a noop + return m; + } + + /// Get a non-owning wrapper of the currently active interpreter (if any) + static subinterpreter current() { + subinterpreter c; + c.istate_ = detail::get_interpreter_state_unchecked(); + c.disarm(); // make destruct a noop, we don't own this... + return c; + } + + /// Get the numerical identifier for the sub-interpreter + int64_t id() const { + if (istate_ != nullptr) { + return PyInterpreterState_GetID(istate_); + } + return -1; // CPython uses one-up numbers from 0, so negative should be safe to return + // here. + } + + /// Get the interpreter's state dict. This interpreter's GIL must be held before calling! + dict state_dict() { return reinterpret_borrow(PyInterpreterState_GetDict(istate_)); } + + /// abandon cleanup of this subinterpreter (leak it). this might be needed during + /// finalization... + void disarm() { creation_tstate_ = nullptr; } + + /// An empty wrapper cannot be activated + bool empty() const { return istate_ == nullptr; } + + /// Is this wrapper non-empty + explicit operator bool() const { return !empty(); } + +private: + friend class subinterpreter_scoped_activate; + friend class subinterpreter_thread_state; + PyInterpreterState *istate_ = nullptr; + PyThreadState *creation_tstate_ = nullptr; +}; + +/// RAII wrapper that owns a PyThreadState bound to a specific subinterpreter on the OS thread +/// that constructed it. Intended to be held long-lived (e.g. as a `thread_local`, or inside a +/// per-thread struct) so that many subinterpreter_scoped_activate scopes on the same OS thread +/// can reuse a single PyThreadState instead of creating and destroying one each time. +/// +/// The PyThreadState is created on construction in a *released* state: it is NOT made current, +/// and no GIL is acquired. Activation is the job of subinterpreter_scoped_activate. +/// +/// A single OS thread can hold one of these per subinterpreter and alternate between them via +/// subinterpreter_scoped_activate without churning PyThreadState objects. +/// +/// Lifetime / threading requirements: +/// +/// - Construction and destruction must happen on the SAME OS thread (a PyThreadState is bound +/// to the OS thread that created it; deleting it on a different thread is undefined behavior). +/// - The owning subinterpreter must still be alive when this object is destroyed. +/// - This object must NOT be destroyed while a subinterpreter_scoped_activate referring to it is +/// still alive (the activator holds a reference into it). +/// +/// Typical usage: +/// +/// @code +/// thread_local py::subinterpreter_thread_state ts(sub); +/// { +/// py::subinterpreter_scoped_activate guard(ts); // swap-in only +/// // ... use the subinterpreter ... +/// } // swap-out, tstate kept alive +/// { +/// py::subinterpreter_scoped_activate guard(ts); // reuses the same PyThreadState +/// // ... +/// } +/// @endcode +class subinterpreter_thread_state { +public: + /// Create a PyThreadState for `si` on the calling OS thread. The new state is left in a + /// released state (not current, no GIL acquired). + explicit subinterpreter_thread_state(subinterpreter const &si); + + /// Destroy the owned PyThreadState. Must run on the same OS thread that constructed this + /// object, while the owning subinterpreter is still alive, and while no + /// subinterpreter_scoped_activate referring to this object is alive. + ~subinterpreter_thread_state(); + + subinterpreter_thread_state(subinterpreter_thread_state const &) = delete; + subinterpreter_thread_state(subinterpreter_thread_state &&) = delete; + subinterpreter_thread_state &operator=(subinterpreter_thread_state const &) = delete; + subinterpreter_thread_state &operator=(subinterpreter_thread_state &&) = delete; + + /// The interpreter this thread state belongs to. + PyInterpreterState *interpreter_state() const { return istate_; } + + /// The owned PyThreadState pointer; valid for the lifetime of this object. + PyThreadState *raw_thread_state() const { return tstate_; } + +private: + friend class subinterpreter_scoped_activate; + PyThreadState *tstate_ = nullptr; + PyInterpreterState *istate_ = nullptr; +}; + +class scoped_subinterpreter { +public: + scoped_subinterpreter() : si_(subinterpreter::create()), scope_(si_) {} + + explicit scoped_subinterpreter(PyInterpreterConfig const &cfg) + : si_(subinterpreter::create(cfg)), scope_(si_) {} + +private: + subinterpreter si_; + subinterpreter_scoped_activate scope_; +}; + +// --- subinterpreter_scoped_activate ----------------------------------------------------------- + +inline subinterpreter_scoped_activate::subinterpreter_scoped_activate(subinterpreter const &si) { + if (!si.istate_) { + pybind11_fail("null subinterpreter"); + } + + if (detail::get_interpreter_state_unchecked() == si.istate_) { + // we are already on this interpreter, make sure we hold the GIL + simple_gil_ = true; + gil_state_ = PyGILState_Ensure(); + return; + } + + // we can't really interact with the interpreter at all until we switch to it + // not even to, for example, look in its state dict or touch its internals + tstate_ = PyThreadState_New(si.istate_); + + // make the interpreter active and acquire the GIL + old_tstate_ = PyThreadState_Swap(tstate_); + + // save this in internals for scoped_gil calls (see also: PR #5870) + detail::get_internals().tstate = tstate_; +} + +inline subinterpreter_scoped_activate::subinterpreter_scoped_activate( + subinterpreter_thread_state &ts) { + if (ts.tstate_ == nullptr) { + pybind11_fail("subinterpreter_scoped_activate: empty subinterpreter_thread_state"); + } + + if (detail::get_interpreter_state_unchecked() == ts.istate_) { + // We are already on this interpreter -- e.g. nested activation, or a different + // PyThreadState for the same interpreter is already current on this thread. Match the + // fast path of the (subinterpreter const&) overload: just ensure the GIL is held. The + // `ts` argument's PyThreadState is intentionally NOT swapped to here; the already-current + // tstate keeps being used until the outer scope exits. + simple_gil_ = true; + gil_state_ = PyGILState_Ensure(); + return; + } + +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) + { + // A PyThreadState is bound to its creating OS thread; it may only be activated there. + bool same_thread = true; +# ifdef PY_HAVE_THREAD_NATIVE_ID + same_thread = PyThread_get_thread_native_id() == ts.tstate_->native_thread_id; +# endif + if (!same_thread) { + pybind11_fail("subinterpreter_scoped_activate: a subinterpreter_thread_state must be " + "activated on the same OS thread that constructed it!"); + } + } +#endif + + tstate_ = ts.tstate_; + borrowed_ = true; + + // make the interpreter active and acquire the GIL + old_tstate_ = PyThreadState_Swap(tstate_); + + // save this in internals for scoped_gil calls (see also: PR #5870) + detail::get_internals().tstate = tstate_; +} + +inline subinterpreter_scoped_activate::~subinterpreter_scoped_activate() { + if (simple_gil_) { + // We were on this interpreter already, so just make sure the GIL goes back as it was + PyGILState_Release(gil_state_); + } else { + if (tstate_) { +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) + if (detail::get_thread_state_unchecked() != tstate_) { + pybind11_fail("~subinterpreter_scoped_activate: thread state must be current!"); + } +#endif + detail::get_internals().tstate.reset(); + if (!borrowed_) { + PyThreadState_Clear(tstate_); + PyThreadState_DeleteCurrent(); + } + // When borrowed_, tstate_ stays alive in its owning subinterpreter_thread_state for + // reuse; the PyThreadState_Swap below merely detaches it from this thread. + } + + // Go back the previous interpreter (if any) and acquire THAT gil + PyThreadState_Swap(old_tstate_); + } +} + +// --- subinterpreter_thread_state -------------------------------------------------------------- + +inline subinterpreter_thread_state::subinterpreter_thread_state(subinterpreter const &si) { + if (!si.istate_) { + pybind11_fail("subinterpreter_thread_state: null subinterpreter"); + } + istate_ = si.istate_; + // PyThreadState_New does not require holding any GIL and does not make the new state current. + tstate_ = PyThreadState_New(istate_); + if (tstate_ == nullptr) { + pybind11_fail("subinterpreter_thread_state: PyThreadState_New returned null"); + } +} + +inline subinterpreter_thread_state::~subinterpreter_thread_state() { + if (tstate_ == nullptr) { + return; + } +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) + { + // A PyThreadState must be cleared and deleted on the OS thread that created it. + bool same_thread = true; +# ifdef PY_HAVE_THREAD_NATIVE_ID + same_thread = PyThread_get_thread_native_id() == tstate_->native_thread_id; +# endif + if (!same_thread) { + pybind11_fail("~subinterpreter_thread_state: must be destroyed on the same OS thread " + "that constructed it!"); + } + } +#endif + // The PyThreadState must be made current to be cleared and deleted on the owning OS thread. + // Swap it in (which acquires the subinterpreter's GIL), clear+delete, then restore whatever + // was active before. + PyThreadState *prev = PyThreadState_Swap(tstate_); + PyThreadState_Clear(tstate_); + PyThreadState_DeleteCurrent(); + // If `prev` is tstate_ itself, the user destroyed this object while it was active via a + // subinterpreter_scoped_activate -- a contract violation, but be defensive: do NOT swap back + // to a now-deleted pointer. Leaving the thread with no current interpreter is consistent + // with the cached state having just been destroyed. + if (prev != nullptr && prev != tstate_) { + PyThreadState_Swap(prev); + } +} + +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/trampoline_self_life_support.h b/lib/pybind11/include/pybind11/trampoline_self_life_support.h new file mode 100644 index 0000000..cbfec7f --- /dev/null +++ b/lib/pybind11/include/pybind11/trampoline_self_life_support.h @@ -0,0 +1,65 @@ +// Copyright (c) 2021 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#pragma once + +#include "detail/common.h" +#include "detail/using_smart_holder.h" +#include "detail/value_and_holder.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_NAMESPACE_BEGIN(detail) +// PYBIND11:REMINDER: Needs refactoring of existing pybind11 code. +inline bool deregister_instance(instance *self, void *valptr, const type_info *tinfo); +PYBIND11_NAMESPACE_END(detail) + +// The original core idea for this struct goes back to PyCLIF: +// https://github.com/google/clif/blob/07f95d7e69dca2fcf7022978a55ef3acff506c19/clif/python/runtime.cc#L37 +// URL provided here mainly to give proper credit. +struct trampoline_self_life_support { + // NOTE: PYBIND11_INTERNALS_VERSION needs to be bumped if changes are made to this struct. + detail::value_and_holder v_h; + + trampoline_self_life_support() = default; + + void activate_life_support(const detail::value_and_holder &v_h_) { + Py_INCREF((PyObject *) v_h_.inst); + v_h = v_h_; + } + + void deactivate_life_support() { + Py_DECREF((PyObject *) v_h.inst); + v_h = detail::value_and_holder(); + } + + ~trampoline_self_life_support() { + if (v_h.inst != nullptr && v_h.vh != nullptr) { + void *value_void_ptr = v_h.value_ptr(); + if (value_void_ptr != nullptr) { + PyGILState_STATE threadstate = PyGILState_Ensure(); + v_h.value_ptr() = nullptr; + v_h.holder().release_disowned(); + detail::deregister_instance(v_h.inst, value_void_ptr, v_h.type); + Py_DECREF((PyObject *) v_h.inst); // Must be after deregister. + PyGILState_Release(threadstate); + } + } + } + + // For the next two, the default implementations generate undefined behavior (ASAN failures + // manually verified). The reason is that v_h needs to be kept default-initialized. + trampoline_self_life_support(const trampoline_self_life_support &) {} + trampoline_self_life_support(trampoline_self_life_support &&) noexcept {} + + // These should never be needed (please provide test cases if you think they are). + trampoline_self_life_support &operator=(const trampoline_self_life_support &) = delete; + trampoline_self_life_support &operator=(trampoline_self_life_support &&) = delete; +}; + +PYBIND11_NAMESPACE_BEGIN(detail) +using get_trampoline_self_life_support_fn = trampoline_self_life_support *(*) (void *); +PYBIND11_NAMESPACE_END(detail) + +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/type_caster_pyobject_ptr.h b/lib/pybind11/include/pybind11/type_caster_pyobject_ptr.h new file mode 100644 index 0000000..aa914f9 --- /dev/null +++ b/lib/pybind11/include/pybind11/type_caster_pyobject_ptr.h @@ -0,0 +1,61 @@ +// Copyright (c) 2023 The pybind Community. + +#pragma once + +#include "detail/common.h" +#include "detail/descr.h" +#include "cast.h" +#include "pytypes.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +template <> +class type_caster { +public: + static constexpr auto name = const_name("object"); // See discussion under PR #4601. + + // This overload is purely to guard against accidents. + template ::value, int> = 0> + static handle cast(T &&, return_value_policy, handle /*parent*/) { + static_assert(is_same_ignoring_cvref::value, + "Invalid C++ type T for to-Python conversion (type_caster)."); + return nullptr; // Unreachable. + } + + static handle cast(PyObject *src, return_value_policy policy, handle /*parent*/) { + if (src == nullptr) { + throw error_already_set(); + } + if (PyErr_Occurred()) { + raise_from(PyExc_SystemError, "src != nullptr but PyErr_Occurred()"); + throw error_already_set(); + } + if (policy == return_value_policy::take_ownership) { + return src; + } + if (policy == return_value_policy::reference + || policy == return_value_policy::automatic_reference) { + return handle(src).inc_ref(); + } + pybind11_fail("type_caster::cast(): unsupported return_value_policy: " + + std::to_string(static_cast(policy))); + } + + bool load(handle src, bool) { + value = reinterpret_borrow(src); + return true; + } + + template + using cast_op_type = PyObject *; + + explicit operator PyObject *() { return value.ptr(); } + +private: + object value; +}; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/typing.h b/lib/pybind11/include/pybind11/typing.h new file mode 100644 index 0000000..4b027db --- /dev/null +++ b/lib/pybind11/include/pybind11/typing.h @@ -0,0 +1,293 @@ +/* + pybind11/typing.h: Convenience wrapper classes for basic Python types + with more explicit annotations. + + Copyright (c) 2023 Dustin Spicuzza + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "detail/common.h" +#include "cast.h" +#include "pytypes.h" + +#include + +#if defined(__cpp_nontype_template_args) && __cpp_nontype_template_args >= 201911L +# define PYBIND11_TYPING_H_HAS_STRING_LITERAL +# include +# include +# include +#endif + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(typing) + +/* + The following types can be used to direct pybind11-generated docstrings + to have have more explicit types (e.g., `list[str]` instead of `list`). + Just use these in place of existing types. + + There is no additional enforcement of types at runtime. +*/ + +// Tuple type hint defined in cast.h for use in py::make_tuple to avoid circular includes + +template +class Dict : public dict { + using dict::dict; +}; + +template +class List : public list { + using list::list; +}; + +template +class Set : public set { + using set::set; +}; + +template +class Iterable : public iterable { + using iterable::iterable; +}; + +template +class Iterator : public iterator { + using iterator::iterator; +}; + +template +class Callable; + +template +class Callable : public function { + using function::function; +}; + +template +class Type : public type { + using type::type; +}; + +template +class Union : public object { + PYBIND11_OBJECT_DEFAULT(Union, object, PyObject_Type) + using object::object; +}; + +template +class Optional : public object { + PYBIND11_OBJECT_DEFAULT(Optional, object, PyObject_Type) + using object::object; +}; + +template +class Final : public object { + PYBIND11_OBJECT_DEFAULT(Final, object, PyObject_Type) + using object::object; +}; + +template +class ClassVar : public object { + PYBIND11_OBJECT_DEFAULT(ClassVar, object, PyObject_Type) + using object::object; +}; + +template +class TypeGuard : public bool_ { + using bool_::bool_; +}; + +template +class TypeIs : public bool_ { + using bool_::bool_; +}; + +class NoReturn : public none { + using none::none; +}; + +class Never : public none { + using none::none; +}; + +#if defined(PYBIND11_TYPING_H_HAS_STRING_LITERAL) +template +struct StringLiteral { + constexpr StringLiteral(const char (&str)[N]) { std::copy_n(str, N, name); } + char name[N]; +}; + +template +class Literal : public object { + PYBIND11_OBJECT_DEFAULT(Literal, object, PyObject_Type) +}; + +// Example syntax for creating a TypeVar. +// typedef typing::TypeVar<"T"> TypeVarT; +template +class TypeVar : public object { + PYBIND11_OBJECT_DEFAULT(TypeVar, object, PyObject_Type) + using object::object; +}; +#endif + +PYBIND11_NAMESPACE_END(typing) + +PYBIND11_NAMESPACE_BEGIN(detail) + +template +struct handle_type_name> { + static constexpr auto name = const_name("tuple[") + + ::pybind11::detail::concat(make_caster::name...) + + const_name("]"); +}; + +template <> +struct handle_type_name> { + // PEP 484 specifies this syntax for an empty tuple + static constexpr auto name = const_name("tuple[()]"); +}; + +template +struct handle_type_name> { + // PEP 484 specifies this syntax for a variable-length tuple + static constexpr auto name + = const_name("tuple[") + make_caster::name + const_name(", ...]"); +}; + +template +struct handle_type_name> { + static constexpr auto name = const_name("dict[") + make_caster::name + const_name(", ") + + make_caster::name + const_name("]"); +}; + +template +struct handle_type_name> { + static constexpr auto name = const_name("list[") + make_caster::name + const_name("]"); +}; + +template +struct handle_type_name> { + static constexpr auto name = const_name("set[") + make_caster::name + const_name("]"); +}; + +template +struct handle_type_name> { + static constexpr auto name + = const_name("collections.abc.Iterable[") + make_caster::name + const_name("]"); +}; + +template +struct handle_type_name> { + static constexpr auto name + = const_name("collections.abc.Iterator[") + make_caster::name + const_name("]"); +}; + +template +struct handle_type_name> { + using retval_type = conditional_t::value, void_type, Return>; + static constexpr auto name + = const_name("collections.abc.Callable[[") + + ::pybind11::detail::concat(::pybind11::detail::inv_descr(make_caster::name)...) + + const_name("], ") + make_caster::name + const_name("]"); +}; + +template +struct handle_type_name> { + // PEP 484 specifies this syntax for defining only return types of callables + using retval_type = conditional_t::value, void_type, Return>; + static constexpr auto name = const_name("collections.abc.Callable[..., ") + + make_caster::name + const_name("]"); +}; + +template +struct handle_type_name> { + static constexpr auto name = const_name("type[") + make_caster::name + const_name("]"); +}; + +template +struct handle_type_name> { + static constexpr auto name = ::pybind11::detail::union_concat(make_caster::name...); +}; + +template +struct handle_type_name> { + static constexpr auto name = make_caster::name | make_caster::name; +}; + +template +struct handle_type_name> { + static constexpr auto name = const_name("typing.Final[") + + ::pybind11::detail::return_descr(make_caster::name) + + const_name("]"); +}; + +template +struct handle_type_name> { + static constexpr auto name + = const_name("typing.ClassVar[") + make_caster::name + const_name("]"); +}; + +template +struct handle_type_name> { + static constexpr auto name = const_name(PYBIND11_TYPE_GUARD_TYPE_HINT) + const_name("[") + + make_caster::name + const_name("]"); +}; + +template +struct handle_type_name> { + static constexpr auto name = const_name(PYBIND11_TYPE_IS_TYPE_HINT) + const_name("[") + + make_caster::name + const_name("]"); +}; + +template <> +struct handle_type_name { + static constexpr auto name = const_name("typing.NoReturn"); +}; + +template <> +struct handle_type_name { + static constexpr auto name = const_name(PYBIND11_NEVER_TYPE_HINT); +}; + +#if defined(PYBIND11_TYPING_H_HAS_STRING_LITERAL) +template +consteval auto sanitize_string_literal() { + constexpr std::string_view v(StrLit.name); + constexpr std::string_view special_chars("!@%{}-"); + constexpr auto num_special_chars = std::accumulate( + special_chars.begin(), special_chars.end(), (size_t) 0, [&v](auto acc, const char &c) { + return std::move(acc) + std::ranges::count(v, c); + }); + char result[v.size() + num_special_chars + 1]; + size_t i = 0; + for (auto c : StrLit.name) { + if (special_chars.find(c) != std::string_view::npos) { + result[i++] = '!'; + } + result[i++] = c; + } + return typing::StringLiteral(result); +} + +template +struct handle_type_name> { + static constexpr auto name + = const_name("typing.Literal[") + + pybind11::detail::concat(const_name(sanitize_string_literal().name)...) + + const_name("]"); +}; +template +struct handle_type_name> { + static constexpr auto name = const_name(sanitize_string_literal().name); +}; +#endif + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/warnings.h b/lib/pybind11/include/pybind11/warnings.h new file mode 100644 index 0000000..263b299 --- /dev/null +++ b/lib/pybind11/include/pybind11/warnings.h @@ -0,0 +1,75 @@ +/* + pybind11/warnings.h: Python warnings wrappers. + + Copyright (c) 2024 Jan Iwaszkiewicz + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "pybind11.h" +#include "detail/common.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_NAMESPACE_BEGIN(detail) + +inline bool PyWarning_Check(PyObject *obj) { + int result = PyObject_IsSubclass(obj, PyExc_Warning); + if (result == 1) { + return true; + } + if (result == -1) { + raise_from(PyExc_SystemError, + "pybind11::detail::PyWarning_Check(): PyObject_IsSubclass() call failed."); + throw error_already_set(); + } + return false; +} + +PYBIND11_NAMESPACE_END(detail) + +PYBIND11_NAMESPACE_BEGIN(warnings) + +inline object +new_warning_type(handle scope, const char *name, handle base = PyExc_RuntimeWarning) { + if (!detail::PyWarning_Check(base.ptr())) { + pybind11_fail("pybind11::warnings::new_warning_type(): cannot create custom warning, base " + "must be a subclass of " + "PyExc_Warning!"); + } + if (hasattr(scope, name)) { + pybind11_fail("pybind11::warnings::new_warning_type(): an attribute with name \"" + + std::string(name) + "\" exists already."); + } + std::string full_name = scope.attr("__name__").cast() + std::string(".") + name; + handle h(PyErr_NewException(full_name.c_str(), base.ptr(), nullptr)); + if (!h) { + raise_from(PyExc_SystemError, + "pybind11::warnings::new_warning_type(): PyErr_NewException() call failed."); + throw error_already_set(); + } + auto obj = reinterpret_steal(h); + scope.attr(name) = obj; + return obj; +} + +// Similar to Python `warnings.warn()` +inline void +warn(const char *message, handle category = PyExc_RuntimeWarning, int stack_level = 2) { + if (!detail::PyWarning_Check(category.ptr())) { + pybind11_fail( + "pybind11::warnings::warn(): cannot raise warning, category must be a subclass of " + "PyExc_Warning!"); + } + + if (PyErr_WarnEx(category.ptr(), message, stack_level) == -1) { + throw error_already_set(); + } +} + +PYBIND11_NAMESPACE_END(warnings) + +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/noxfile.py b/lib/pybind11/noxfile.py deleted file mode 100644 index 021ced2..0000000 --- a/lib/pybind11/noxfile.py +++ /dev/null @@ -1,107 +0,0 @@ -import os - -import nox - -nox.needs_version = ">=2022.1.7" -nox.options.sessions = ["lint", "tests", "tests_packaging"] - -PYTHON_VERSIONS = [ - "3.6", - "3.7", - "3.8", - "3.9", - "3.10", - "3.11", - "pypy3.7", - "pypy3.8", - "pypy3.9", -] - -if os.environ.get("CI", None): - nox.options.error_on_missing_interpreters = True - - -@nox.session(reuse_venv=True) -def lint(session: nox.Session) -> None: - """ - Lint the codebase (except for clang-format/tidy). - """ - session.install("pre-commit") - session.run("pre-commit", "run", "-a", *session.posargs) - - -@nox.session(python=PYTHON_VERSIONS) -def tests(session: nox.Session) -> None: - """ - Run the tests (requires a compiler). - """ - tmpdir = session.create_tmp() - session.install("cmake") - session.install("-r", "tests/requirements.txt") - session.run( - "cmake", - "-S.", - f"-B{tmpdir}", - "-DPYBIND11_WERROR=ON", - "-DDOWNLOAD_CATCH=ON", - "-DDOWNLOAD_EIGEN=ON", - *session.posargs, - ) - session.run("cmake", "--build", tmpdir) - session.run("cmake", "--build", tmpdir, "--config=Release", "--target", "check") - - -@nox.session -def tests_packaging(session: nox.Session) -> None: - """ - Run the packaging tests. - """ - - session.install("-r", "tests/requirements.txt", "--prefer-binary") - session.run("pytest", "tests/extra_python_package", *session.posargs) - - -@nox.session(reuse_venv=True) -def docs(session: nox.Session) -> None: - """ - Build the docs. Pass "serve" to serve. - """ - - session.install("-r", "docs/requirements.txt") - session.chdir("docs") - - if "pdf" in session.posargs: - session.run("sphinx-build", "-M", "latexpdf", ".", "_build") - return - - session.run("sphinx-build", "-M", "html", ".", "_build") - - if "serve" in session.posargs: - session.log("Launching docs at http://localhost:8000/ - use Ctrl-C to quit") - session.run("python", "-m", "http.server", "8000", "-d", "_build/html") - elif session.posargs: - session.error("Unsupported argument to docs") - - -@nox.session(reuse_venv=True) -def make_changelog(session: nox.Session) -> None: - """ - Inspect the closed issues and make entries for a changelog. - """ - session.install("ghapi", "rich") - session.run("python", "tools/make_changelog.py") - - -@nox.session(reuse_venv=True) -def build(session: nox.Session) -> None: - """ - Build SDists and wheels. - """ - - session.install("build") - session.log("Building normal files") - session.run("python", "-m", "build", *session.posargs) - session.log("Building pybind11-global files (PYBIND11_GLOBAL_SDIST=1)") - session.run( - "python", "-m", "build", *session.posargs, env={"PYBIND11_GLOBAL_SDIST": "1"} - ) diff --git a/lib/pybind11/pybind11/__init__.py b/lib/pybind11/pybind11/__init__.py index 4fbb170..3882b2b 100644 --- a/lib/pybind11/pybind11/__init__.py +++ b/lib/pybind11/pybind11/__init__.py @@ -1,7 +1,9 @@ +from __future__ import annotations + import sys -if sys.version_info < (3, 6): - msg = "pybind11 does not support Python < 3.6. 2.9 was the last release supporting Python 2.7 and 3.5." +if sys.version_info < (3, 9): # noqa: UP036 + msg = "pybind11 does not support Python < 3.9. v3.0 was the last release supporting Python 3.8." raise ImportError(msg) @@ -9,9 +11,9 @@ from .commands import get_cmake_dir, get_include, get_pkgconfig_dir __all__ = ( - "version_info", "__version__", - "get_include", "get_cmake_dir", + "get_include", "get_pkgconfig_dir", + "version_info", ) diff --git a/lib/pybind11/pybind11/__main__.py b/lib/pybind11/pybind11/__main__.py index 8c89533..ce597c7 100644 --- a/lib/pybind11/pybind11/__main__.py +++ b/lib/pybind11/pybind11/__main__.py @@ -1,31 +1,40 @@ # pylint: disable=missing-function-docstring +from __future__ import annotations import argparse +import functools import sys import sysconfig +from pathlib import Path -from .commands import get_cmake_dir, get_include, get_pkgconfig_dir +from ._version import __version__ +from .commands import ( + _quote as quote, +) +from .commands import ( + get_cflags, + get_cmake_dir, + get_include_dirs, + get_ldflags, + get_pkgconfig_dir, +) def print_includes() -> None: - dirs = [ - sysconfig.get_path("include"), - sysconfig.get_path("platinclude"), - get_include(), - ] - - # Make unique but preserve order - unique_dirs = [] - for d in dirs: - if d and d not in unique_dirs: - unique_dirs.append(d) - - print(" ".join("-I" + d for d in unique_dirs)) + print(" ".join(quote(f"-I{d}") for d in get_include_dirs())) def main() -> None: - - parser = argparse.ArgumentParser() + make_parser = functools.partial(argparse.ArgumentParser, allow_abbrev=False) + if sys.version_info >= (3, 14): + make_parser = functools.partial(make_parser, color=True, suggest_on_error=True) + parser = make_parser() + parser.add_argument( + "--version", + action="version", + version=__version__, + help="Print the version and exit.", + ) parser.add_argument( "--includes", action="store_true", @@ -41,15 +50,59 @@ def main() -> None: action="store_true", help="Print the pkgconfig directory, ideal for setting $PKG_CONFIG_PATH.", ) + parser.add_argument( + "--extension-suffix", + action="store_true", + help="Print the extension for a Python module", + ) + parser.add_argument( + "--cflags", + action="store_true", + help="Print the compile flags for a simple extension (Unix-style compilers).", + ) + parser.add_argument( + "--ldflags", + action="store_true", + help="Print the link flags for a simple extension (Unix-style compilers).", + ) + parser.add_argument( + "--embed", + action="store_true", + help="Build for embedding instead of an extension; affects --ldflags and --file.", + ) + parser.add_argument( + "--file", + type=Path, + help="Print a full command-line suffix for compiling the given file;" + " the output goes next to the source file (Unix-style compilers).", + ) args = parser.parse_args() if not sys.argv[1:]: parser.print_help() - if args.includes: + ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") or "" + if args.file: + suffix = "" if args.embed else ext_suffix + print( + get_cflags(), + quote(str(args.file)), + get_ldflags(embed=args.embed), + "-o", + quote(str(args.file.with_suffix(suffix))), + ) + else: + if args.cflags: + print(get_cflags()) + if args.ldflags: + print(get_ldflags(embed=args.embed)) + # --cflags and --file already contain the include flags + if args.includes and not (args.cflags or args.file): print_includes() if args.cmakedir: - print(get_cmake_dir()) + print(quote(get_cmake_dir())) if args.pkgconfigdir: - print(get_pkgconfig_dir()) + print(quote(get_pkgconfig_dir())) + if args.extension_suffix: + print(ext_suffix) if __name__ == "__main__": diff --git a/lib/pybind11/pybind11/_version.py b/lib/pybind11/pybind11/_version.py index 1cb51fc..d31cadb 100644 --- a/lib/pybind11/pybind11/_version.py +++ b/lib/pybind11/pybind11/_version.py @@ -1,12 +1,34 @@ -from typing import Union +# This file will be replaced in the wheel with a hard-coded version. This only +# exists to allow running directly from source without installing (not +# recommended, but supported). +from __future__ import annotations -def _to_int(s: str) -> Union[int, str]: +import re +from pathlib import Path + +DIR = Path(__file__).parent.resolve() + +input_file = DIR.parent / "include/pybind11/detail/common.h" +regex = re.compile( + r""" +\#define \s+ PYBIND11_VERSION_MAJOR \s+ (?P\d+) .*? +\#define \s+ PYBIND11_VERSION_MINOR \s+ (?P\d+) .*? +\#define \s+ PYBIND11_VERSION_PATCH \s+ (?P\S+) +""", + re.MULTILINE | re.DOTALL | re.VERBOSE, +) + +match = regex.search(input_file.read_text(encoding="utf-8")) +assert match, "Unable to find version in pybind11/detail/common.h" +__version__ = "{major}.{minor}.{patch}".format(**match.groupdict()) + + +def _to_int(s: str) -> int | str: try: return int(s) except ValueError: return s -__version__ = "2.11.0.dev1" version_info = tuple(_to_int(s) for s in __version__.split(".")) diff --git a/lib/pybind11/pybind11/commands.py b/lib/pybind11/pybind11/commands.py index 152fa20..8bd0a9b 100644 --- a/lib/pybind11/pybind11/commands.py +++ b/lib/pybind11/pybind11/commands.py @@ -1,9 +1,48 @@ +from __future__ import annotations + import os +import re +import sys DIR = os.path.abspath(os.path.dirname(__file__)) +# shlex/sysconfig are imported lazily below to keep `import pybind11` light + +# This is the conditional used for os.path being posixpath +if "posix" in sys.builtin_module_names: + + def _quote(s: str) -> str: + import shlex + + return shlex.quote(s) +elif "nt" in sys.builtin_module_names: + # See https://github.com/mesonbuild/meson/blob/db22551ed9d2dd7889abea01cc1c7bba02bf1c75/mesonbuild/utils/universal.py#L1092-L1121 + # and the original documents: + # https://docs.microsoft.com/en-us/cpp/c-language/parsing-c-command-line-arguments and + # https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/ + _UNSAFE = re.compile("[ \t\n\r]") + + def _quote(s: str) -> str: + if s and not _UNSAFE.search(s): + return s + + # Paths cannot contain a '"' on Windows, so we don't need to worry + # about nuanced counting here. + return f'"{s}\\"' if s.endswith("\\") else f'"{s}"' +else: + + def _quote(s: str) -> str: + return s -def get_include(user: bool = False) -> str: # pylint: disable=unused-argument + +def _config(name: str) -> str: + """A stripped sysconfig variable, or "" if unset.""" + import sysconfig + + return str(sysconfig.get_config_var(name) or "").strip() + + +def get_include(user: bool = False) -> str: # noqa: ARG001 """ Return the path to the pybind11 include directory. The historical "user" argument is unused, and may be removed. @@ -35,3 +74,67 @@ def get_pkgconfig_dir() -> str: msg = "pybind11 not installed, installation required to access the pkgconfig files" raise ImportError(msg) + + +def get_include_dirs() -> list[str]: + """ + Return the unique include directories for Python and pybind11. + """ + import sysconfig + + dirs = [ + sysconfig.get_path("include"), + sysconfig.get_path("platinclude"), + get_include(), + ] + + # Make unique but preserve order + unique_dirs = [] + for d in dirs: + if d and d not in unique_dirs: + unique_dirs.append(d) + return unique_dirs + + +def get_cflags() -> str: + """ + Return the compile flags for building a simple extension with a + Unix-style command-line compiler (GCC/Clang). Based on python-config. + """ + flags = [_quote(f"-I{d}") for d in get_include_dirs()] + # CFLAGS is a pre-composed multi-flag string, passed through as-is (like + # python-config does) + if cflags := _config("CFLAGS"): + flags.append(cflags) + flags.append("-std=c++17") + return " ".join(flags) + + +def get_ldflags(embed: bool = False) -> str: + """ + Return the link flags for building a simple extension (or, with + embed=True, an embedding program) with a Unix-style command-line + compiler (GCC/Clang). Based on python-config. + """ + # LDFLAGS/LIBS/SYSLIBS are pre-composed multi-flag strings, passed + # through as-is (like python-config does) + flags = [ldflags] if (ldflags := _config("LDFLAGS")) else [] + + if embed: + if libdir := _config("LIBDIR"): + flags.append(_quote(f"-L{libdir}")) + if not _config("Py_ENABLE_SHARED") and (libpl := _config("LIBPL")): + flags.append(_quote(f"-L{libpl}")) + abiflags = getattr(sys, "abiflags", "") or "" + flags.append(f"-lpython{_config('VERSION')}{abiflags}") + if libs := _config("LIBS"): + flags.append(libs) + if syslibs := _config("SYSLIBS"): + flags.append(syslibs) + elif sys.platform.startswith("darwin"): + flags += ["-undefined", "dynamic_lookup", "-shared"] + elif os.name == "posix": + # Linux and other Unix platforms (FreeBSD, Solaris, AIX, ...) + flags += ["-fPIC", "-shared"] + + return " ".join(flags) diff --git a/lib/pybind11/pybind11/setup_helpers.py b/lib/pybind11/pybind11/setup_helpers.py index 1fd04b9..66a3eec 100644 --- a/lib/pybind11/pybind11/setup_helpers.py +++ b/lib/pybind11/pybind11/setup_helpers.py @@ -36,6 +36,7 @@ # # If you copy this file in, you don't # need the .pyi file; it's just an interface file for static type checkers. +from __future__ import annotations import contextlib import os @@ -47,18 +48,14 @@ import tempfile import threading import warnings +from collections.abc import Iterable, Iterator from functools import lru_cache from pathlib import Path from typing import ( + TYPE_CHECKING, Any, Callable, - Dict, - Iterable, - Iterator, - List, Optional, - Tuple, - TypeVar, Union, ) @@ -66,12 +63,17 @@ from setuptools import Extension as _Extension from setuptools.command.build_ext import build_ext as _build_ext except ImportError: - from distutils.command.build_ext import build_ext as _build_ext - from distutils.extension import Extension as _Extension + from distutils.command.build_ext import ( # type: ignore[assignment] + build_ext as _build_ext, + ) + from distutils.extension import Extension as _Extension # type: ignore[assignment] import distutils.ccompiler import distutils.errors +if TYPE_CHECKING: + from typing_extensions import Self + WIN = sys.platform.startswith("win32") and "mingw" not in sysconfig.get_platform() MACOS = sys.platform.startswith("darwin") STD_TMPL = "/std:c++{}" if WIN else "-std=c++{}" @@ -84,7 +86,7 @@ # directory into your path if it sits beside your setup.py. -class Pybind11Extension(_Extension): # type: ignore[misc] +class Pybind11Extension(_Extension): """ Build a C++11+ Extension module with pybind11. This automatically adds the recommended flags when you init the extension and assumes C++ sources - you @@ -111,14 +113,13 @@ class Pybind11Extension(_Extension): # type: ignore[misc] # flags are prepended, so that they can be further overridden, e.g. by # ``extra_compile_args=["-g"]``. - def _add_cflags(self, flags: List[str]) -> None: + def _add_cflags(self, flags: list[str]) -> None: self.extra_compile_args[:0] = flags - def _add_ldflags(self, flags: List[str]) -> None: + def _add_ldflags(self, flags: list[str]) -> None: self.extra_link_args[:0] = flags def __init__(self, *args: Any, **kwargs: Any) -> None: - self._cxx_level = 0 cxx_std = kwargs.pop("cxx_std", 0) @@ -145,7 +146,6 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self.cxx_std = cxx_std cflags = [] - ldflags = [] if WIN: cflags += ["/EHsc", "/bigobj"] else: @@ -155,11 +155,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: c_cpp_flags = shlex.split(env_cflags) + shlex.split(env_cppflags) if not any(opt.startswith("-g") for opt in c_cpp_flags): cflags += ["-g0"] - if MACOS: - cflags += ["-stdlib=libc++"] - ldflags += ["-stdlib=libc++"] self._add_cflags(cflags) - self._add_ldflags(ldflags) @property def cxx_std(self) -> int: @@ -174,12 +170,12 @@ def cxx_std(self) -> int: @cxx_std.setter def cxx_std(self, level: int) -> None: - if self._cxx_level: - warnings.warn("You cannot safely change the cxx_level after setting it!") + warnings.warn( + "You cannot safely change the cxx_level after setting it!", stacklevel=2 + ) - # MSVC 2015 Update 3 and later only have 14 (and later 17) modes, so - # force a valid flag here. + # MSVC only has 14 and later modes, so force a valid flag here. if WIN and level == 11: level = 14 @@ -248,12 +244,8 @@ def has_flag(compiler: Any, flag: str) -> bool: return True -# Every call will cache the result -cpp_flag_cache = None - - -@lru_cache() -def auto_cpp_level(compiler: Any) -> Union[str, int]: +@lru_cache +def auto_cpp_level(compiler: Any) -> str | int: """ Return the max supported C++ std level (17, 14, or 11). Returns latest on Windows. """ @@ -271,7 +263,7 @@ def auto_cpp_level(compiler: Any) -> Union[str, int]: raise RuntimeError(msg) -class build_ext(_build_ext): # type: ignore[misc] # noqa: N801 +class build_ext(_build_ext): # noqa: N801 """ Customized build_ext that allows an auto-search for the highest supported C++ level for Pybind11Extension. This is only needed for the auto-search @@ -291,8 +283,8 @@ def build_extensions(self) -> None: def intree_extensions( - paths: Iterable[str], package_dir: Optional[Dict[str, str]] = None -) -> List[Pybind11Extension]: + paths: Iterable[str], package_dir: dict[str, str] | None = None +) -> list[Pybind11Extension]: """ Generate Pybind11Extensions from source files directly located in a Python source tree. @@ -341,7 +333,7 @@ def naive_recompile(obj: str, src: str) -> bool: return os.stat(obj).st_mtime < os.stat(src).st_mtime -def no_recompile(obg: str, src: str) -> bool: # pylint: disable=unused-argument +def no_recompile(obj: str, src: str) -> bool: # noqa: ARG001 """ This is the safest but slowest choice (and is the default) - will always recompile sources. @@ -349,21 +341,19 @@ def no_recompile(obg: str, src: str) -> bool: # pylint: disable=unused-argument return True -S = TypeVar("S", bound="ParallelCompile") - CCompilerMethod = Callable[ [ distutils.ccompiler.CCompiler, - List[str], + list[str], Optional[str], - Optional[Union[Tuple[str], Tuple[str, Optional[str]]]], - Optional[List[str]], + Optional[list[Union[tuple[str], tuple[str, Optional[str]]]]], + Optional[list[str]], bool, - Optional[List[str]], - Optional[List[str]], - Optional[List[str]], + Optional[list[str]], + Optional[list[str]], + Optional[list[str]], ], - List[str], + list[str], ] @@ -408,11 +398,11 @@ class ParallelCompile: called. """ - __slots__ = ("envvar", "default", "max", "_old", "needs_recompile") + __slots__ = ("_old", "default", "envvar", "max", "needs_recompile") def __init__( self, - envvar: Optional[str] = None, + envvar: str | None = None, default: int = 0, max: int = 0, # pylint: disable=redefined-builtin needs_recompile: Callable[[str, str], bool] = no_recompile, @@ -421,7 +411,7 @@ def __init__( self.default = default self.max = max self.needs_recompile = needs_recompile - self._old: List[CCompilerMethod] = [] + self._old: list[CCompilerMethod] = [] def function(self) -> CCompilerMethod: """ @@ -430,16 +420,15 @@ def function(self) -> CCompilerMethod: def compile_function( compiler: distutils.ccompiler.CCompiler, - sources: List[str], - output_dir: Optional[str] = None, - macros: Optional[Union[Tuple[str], Tuple[str, Optional[str]]]] = None, - include_dirs: Optional[List[str]] = None, + sources: list[str], + output_dir: str | None = None, + macros: list[tuple[str] | tuple[str, str | None]] | None = None, + include_dirs: list[str] | None = None, debug: bool = False, - extra_preargs: Optional[List[str]] = None, - extra_postargs: Optional[List[str]] = None, - depends: Optional[List[str]] = None, + extra_preargs: list[str] | None = None, + extra_postargs: list[str] | None = None, + depends: list[str] | None = None, ) -> Any: - # These lines are directly from distutils.ccompiler.CCompiler macros, objects, extra_postargs, pp_opts, build = compiler._setup_compile( # type: ignore[attr-defined] output_dir, macros, include_dirs, sources, depends, extra_postargs @@ -489,16 +478,16 @@ def _single_compile(obj: Any) -> None: return compile_function - def install(self: S) -> S: + def install(self) -> Self: """ Installs the compile function into distutils.ccompiler.CCompiler.compile. """ distutils.ccompiler.CCompiler.compile = self.function() # type: ignore[assignment] return self - def __enter__(self: S) -> S: + def __enter__(self) -> Self: self._old.append(distutils.ccompiler.CCompiler.compile) return self.install() - def __exit__(self, *args: Any) -> None: + def __exit__(self, *args: object) -> None: distutils.ccompiler.CCompiler.compile = self._old.pop() # type: ignore[assignment] diff --git a/lib/pybind11/pyproject.toml b/lib/pybind11/pyproject.toml index 3ba1b4b..cb5d728 100644 --- a/lib/pybind11/pyproject.toml +++ b/lib/pybind11/pyproject.toml @@ -1,53 +1,136 @@ [build-system] -requires = ["setuptools>=42", "cmake>=3.18", "ninja"] -build-backend = "setuptools.build_meta" +requires = ["scikit-build-core >=0.11.2"] +build-backend = "scikit_build_core.build" + +[project] +name = "pybind11" +description = "Seamless operability between C++11 and Python" +authors = [{name = "Wenzel Jakob", email = "wenzel.jakob@epfl.ch"}] +license = "BSD-3-Clause" +license-files = ["LICENSE"] +readme = "README.rst" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Utilities", + "Programming Language :: C++", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", + "Programming Language :: Python :: Implementation :: PyPy", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: C++", + "Topic :: Software Development :: Libraries :: Python Modules", +] +keywords = [ + "C++11", + "Python bindings", +] +dynamic = ["version", "optional-dependencies"] +requires-python = ">=3.9" + +[project.urls] +Homepage = "https://github.com/pybind/pybind11" +Documentation = "https://pybind11.readthedocs.io/" +"Issue Tracker" = "https://github.com/pybind/pybind11/issues" +Discussions = "https://github.com/pybind/pybind11/discussions" +Changelog = "https://pybind11.readthedocs.io/en/latest/changelog.html" +Chat = "https://gitter.im/pybind/Lobby" + +[project.scripts] +pybind11-config = "pybind11.__main__:main" + +[project.entry-points."pipx.run"] +pybind11 = "pybind11.__main__:main" + +[project.entry-points.pkg_config] +pybind11 = "pybind11.share.pkgconfig" + + +[dependency-groups] +test = [ + "pytest", + "build", +] +dev = [ + "tomlkit", + { include-group = "test" } +] + + +[tool.scikit-build] +minimum-version = "build-system.requires" +sdist.exclude = [ + "/docs/**", + "/.**", +] +wheel.install-dir = "pybind11" +wheel.platlib = false + +[tool.scikit-build.cmake.define] +BUILD_TESTING = false +PYBIND11_NOPYTHON = true +prefix_for_pc_file = "${pcfiledir}/../../" + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "include/pybind11/detail/common.h" +regex = '''(?sx) +\#define \s+ PYBIND11_VERSION_MAJOR \s+ (?P\d+) .*? +\#define \s+ PYBIND11_VERSION_MINOR \s+ (?P\d+) .*? +\#define \s+ PYBIND11_VERSION_PATCH \s+ (?P\S+) +''' +result = "{major}.{minor}.{patch}" + +[tool.scikit-build.metadata.optional-dependencies] +provider = "scikit_build_core.metadata.template" +result = { global = ["pybind11-global=={project[version]}"]} + +[[tool.scikit-build.generate]] +path = "pybind11/_version.py" +template = ''' +from __future__ import annotations + + +def _to_int(s: str) -> int | str: + try: + return int(s) + except ValueError: + return s + + +__version__ = "$version" +version_info = tuple(_to_int(s) for s in __version__.split(".")) +''' + + +[tool.uv] +# Can't use tool.uv.sources with requirements.txt +index-strategy = "unsafe-best-match" +# This extra confuses uv +override-dependencies = ["pybind11-global"] -[tool.check-manifest] -ignore = [ - "tests/**", - "docs/**", - "tools/**", - "include/**", - ".*", - "pybind11/include/**", - "pybind11/share/**", - "CMakeLists.txt", - "noxfile.py", -] - -[tool.isort] -# Needs the compiled .so modules and env.py from tests -known_first_party = "env,pybind11_cross_module_tests,pybind11_tests," -# For black compatibility -profile = "black" [tool.mypy] files = ["pybind11"] -python_version = "3.6" +python_version = "3.9" strict = true -show_error_codes = true enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] warn_unreachable = true [[tool.mypy.overrides]] -module = ["ghapi.*", "setuptools.*"] +module = ["ghapi.*", "tomlkit"] # tomlkit has types, but not very helpful ignore_missing_imports = true -[tool.pytest.ini_options] -minversion = "6.0" -addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config"] -xfail_strict = true -filterwarnings = ["error"] -log_cli_level = "info" -testpaths = [ - "tests", -] -timeout=300 - - [tool.pylint] -master.py-version = "3.6" +master.py-version = "3.9" reports.output-format = "colorized" messages_control.disable = [ "design", @@ -58,4 +141,81 @@ messages_control.disable = [ "invalid-name", "protected-access", "missing-module-docstring", + "unused-argument", # covered by Ruff ARG + "consider-using-f-string", # triggers in _version.py incorrectly +] + +[tool.ruff.lint] +extend-select = [ + "B", # flake8-bugbear + "I", # isort + "N", # pep8-naming + "ARG", # flake8-unused-arguments + "C4", # flake8-comprehensions + "EM", # flake8-errmsg + "ICN", # flake8-import-conventions + "ISC", # flake8-implicit-str-concat + "PERF", # Perflint + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PL", # pylint + "PT", # flake8-pytest-style + "RET", # flake8-return + "RUF100", # Ruff-specific + "SIM", # flake8-simplify + "UP", # pyupgrade + "YTT", # flake8-2020 +] +ignore = [ + "PLR", # Design related pylint + "PT011", # Too broad with raises in pytest + "SIM118", # iter(x) is not always the same as iter(x.keys()) + "PLC0415", # We import in functions for various reasons ] +isort.known-first-party = ["env", "pybind11_cross_module_tests", "pybind11_tests"] +isort.required-imports = ["from __future__ import annotations"] + + +[tool.ruff.lint.per-file-ignores] +"tests/**" = [ + "EM", + "N", + "E721", + "BLE001", # Tests capture and re-report exceptions from workers and callbacks + "DTZ", # test_chrono.py checks naive local-time round-tripping on purpose + "FLY002", # Joining a list keeps long signatures one-per-line + "RUF012", # ClassVar annotations are noise in test fixtures + "RUF063", # test_pytypes.py reads __annotations__ from __dict__ deliberately +] +"tests/test_call_policies.py" = ["PLC1901"] +"docs/benchmark.py" = ["DTZ"] + +[tool.repo-review] +ignore = ["PP"] + +[tool.typos] +files.extend-exclude = ["/cpython"] + +[tool.typos.default.extend-identifiers] +ser_no = "ser_no" +SerNo = "SerNo" +StrLits = "StrLits" + +[tool.typos.default.extend-words] +nd = "nd" +valu = "valu" +fo = "fo" +quater = "quater" +optin = "optin" +othr = "othr" +# NumPy uses "writeable" in public API names and flags. +writeable = "writeable" +Writeable = "Writeable" +WRITEABLE = "WRITEABLE" + +#[tool.typos.type.cpp.extend-words] +setp = "setp" +ot = "ot" + +[tool.typos.type.json.extend-words] +ba = "ba" diff --git a/lib/pybind11/setup.cfg b/lib/pybind11/setup.cfg deleted file mode 100644 index 8b33619..0000000 --- a/lib/pybind11/setup.cfg +++ /dev/null @@ -1,50 +0,0 @@ -[metadata] -long_description = file: README.rst -long_description_content_type = text/x-rst -description = Seamless operability between C++11 and Python -author = Wenzel Jakob -author_email = wenzel.jakob@epfl.ch -url = https://github.com/pybind/pybind11 -license = BSD - -classifiers = - Development Status :: 5 - Production/Stable - Intended Audience :: Developers - Topic :: Software Development :: Libraries :: Python Modules - Topic :: Utilities - Programming Language :: C++ - Programming Language :: Python :: 3 :: Only - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Programming Language :: Python :: 3.11 - License :: OSI Approved :: BSD License - Programming Language :: Python :: Implementation :: PyPy - Programming Language :: Python :: Implementation :: CPython - Programming Language :: C++ - Topic :: Software Development :: Libraries :: Python Modules - -keywords = - C++11 - Python bindings - -project_urls = - Documentation = https://pybind11.readthedocs.io/ - Bug Tracker = https://github.com/pybind/pybind11/issues - Discussions = https://github.com/pybind/pybind11/discussions - Changelog = https://pybind11.readthedocs.io/en/latest/changelog.html - Chat = https://gitter.im/pybind/Lobby - -[options] -python_requires = >=3.6 -zip_safe = False - - -[flake8] -max-line-length = 120 -show_source = True -exclude = .git, __pycache__, build, dist, docs, tools, venv -extend-ignore = E203, E722 -extend-select = B902, B904 diff --git a/lib/pybind11/setup.py b/lib/pybind11/setup.py deleted file mode 100644 index 6857351..0000000 --- a/lib/pybind11/setup.py +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env python3 - -# Setup script for PyPI; use CMakeFile.txt to build extension modules - -import contextlib -import os -import re -import shutil -import string -import subprocess -import sys -from pathlib import Path -from tempfile import TemporaryDirectory -from typing import Dict, Iterator, List, Union - -import setuptools.command.sdist - -DIR = Path(__file__).parent.absolute() -VERSION_REGEX = re.compile( - r"^\s*#\s*define\s+PYBIND11_VERSION_([A-Z]+)\s+(.*)$", re.MULTILINE -) -VERSION_FILE = Path("pybind11/_version.py") -COMMON_FILE = Path("include/pybind11/detail/common.h") - - -def build_expected_version_hex(matches: Dict[str, str]) -> str: - patch_level_serial = matches["PATCH"] - serial = None - major = int(matches["MAJOR"]) - minor = int(matches["MINOR"]) - flds = patch_level_serial.split(".") - if flds: - patch = int(flds[0]) - if len(flds) == 1: - level = "0" - serial = 0 - elif len(flds) == 2: - level_serial = flds[1] - for level in ("a", "b", "c", "dev"): - if level_serial.startswith(level): - serial = int(level_serial[len(level) :]) - break - if serial is None: - msg = f'Invalid PYBIND11_VERSION_PATCH: "{patch_level_serial}"' - raise RuntimeError(msg) - version_hex_str = f"{major:02x}{minor:02x}{patch:02x}{level[:1]}{serial:x}" - return f"0x{version_hex_str.upper()}" - - -# PYBIND11_GLOBAL_SDIST will build a different sdist, with the python-headers -# files, and the sys.prefix files (CMake and headers). - -global_sdist = os.environ.get("PYBIND11_GLOBAL_SDIST", False) - -setup_py = Path( - "tools/setup_global.py.in" if global_sdist else "tools/setup_main.py.in" -) -extra_cmd = 'cmdclass["sdist"] = SDist\n' - -to_src = ( - (Path("pyproject.toml"), Path("tools/pyproject.toml")), - (Path("setup.py"), setup_py), -) - - -# Read the listed version -loc: Dict[str, str] = {} -code = compile(VERSION_FILE.read_text(encoding="utf-8"), "pybind11/_version.py", "exec") -exec(code, loc) -version = loc["__version__"] - -# Verify that the version matches the one in C++ -matches = dict(VERSION_REGEX.findall(COMMON_FILE.read_text(encoding="utf8"))) -cpp_version = "{MAJOR}.{MINOR}.{PATCH}".format(**matches) -if version != cpp_version: - msg = f"Python version {version} does not match C++ version {cpp_version}!" - raise RuntimeError(msg) - -version_hex = matches.get("HEX", "MISSING") -exp_version_hex = build_expected_version_hex(matches) -if version_hex != exp_version_hex: - msg = f"PYBIND11_VERSION_HEX {version_hex} does not match expected value {exp_version_hex}!" - raise RuntimeError(msg) - - -# TODO: use literals & overload (typing extensions or Python 3.8) -def get_and_replace( - filename: Path, binary: bool = False, **opts: str -) -> Union[bytes, str]: - if binary: - contents = filename.read_bytes() - return string.Template(contents.decode()).substitute(opts).encode() - - return string.Template(filename.read_text()).substitute(opts) - - -# Use our input files instead when making the SDist (and anything that depends -# on it, like a wheel) -class SDist(setuptools.command.sdist.sdist): # type: ignore[misc] - def make_release_tree(self, base_dir: str, files: List[str]) -> None: - super().make_release_tree(base_dir, files) - - for to, src in to_src: - txt = get_and_replace(src, binary=True, version=version, extra_cmd="") - - dest = Path(base_dir) / to - - # This is normally linked, so unlink before writing! - dest.unlink() - dest.write_bytes(txt) # type: ignore[arg-type] - - -# Remove the CMake install directory when done -@contextlib.contextmanager -def remove_output(*sources: str) -> Iterator[None]: - try: - yield - finally: - for src in sources: - shutil.rmtree(src) - - -with remove_output("pybind11/include", "pybind11/share"): - # Generate the files if they are not present. - with TemporaryDirectory() as tmpdir: - cmd = ["cmake", "-S", ".", "-B", tmpdir] + [ - "-DCMAKE_INSTALL_PREFIX=pybind11", - "-DBUILD_TESTING=OFF", - "-DPYBIND11_NOPYTHON=ON", - "-Dprefix_for_pc_file=${pcfiledir}/../../", - ] - if "CMAKE_ARGS" in os.environ: - fcommand = [ - c - for c in os.environ["CMAKE_ARGS"].split() - if "DCMAKE_INSTALL_PREFIX" not in c - ] - cmd += fcommand - subprocess.run(cmd, check=True, cwd=DIR, stdout=sys.stdout, stderr=sys.stderr) - subprocess.run( - ["cmake", "--install", tmpdir], - check=True, - cwd=DIR, - stdout=sys.stdout, - stderr=sys.stderr, - ) - - txt = get_and_replace(setup_py, version=version, extra_cmd=extra_cmd) - code = compile(txt, setup_py, "exec") - exec(code, {"SDist": SDist}) diff --git a/lib/pybind11/tools/FindCatch.cmake b/lib/pybind11/tools/FindCatch.cmake index 57bba58..967434c 100644 --- a/lib/pybind11/tools/FindCatch.cmake +++ b/lib/pybind11/tools/FindCatch.cmake @@ -9,8 +9,6 @@ # CATCH_INCLUDE_DIR - path to catch.hpp # CATCH_VERSION - version number -option(DOWNLOAD_CATCH "Download catch2 if not found") - if(NOT Catch_FIND_VERSION) message(FATAL_ERROR "A version number must be specified.") elseif(Catch_FIND_REQUIRED) @@ -36,10 +34,14 @@ endfunction() function(_download_catch version destination_dir) message(STATUS "Downloading catch v${version}...") set(url https://github.com/philsquared/Catch/releases/download/v${version}/catch.hpp) - file(DOWNLOAD ${url} "${destination_dir}/catch.hpp" STATUS status) + file( + DOWNLOAD ${url} "${destination_dir}/catch.hpp" + STATUS status + LOG log) list(GET status 0 error) if(error) - message(FATAL_ERROR "Could not download ${url}") + string(REPLACE "\n" "\n " log " ${log}") + message(FATAL_ERROR "Could not download URL:\n" " ${url}\n" "Log:\n" "${log}") endif() set(CATCH_INCLUDE_DIR "${destination_dir}" diff --git a/lib/pybind11/tools/FindPythonLibsNew.cmake b/lib/pybind11/tools/FindPythonLibsNew.cmake index ce558d4..d72e213 100644 --- a/lib/pybind11/tools/FindPythonLibsNew.cmake +++ b/lib/pybind11/tools/FindPythonLibsNew.cmake @@ -92,7 +92,23 @@ endif() # Use the Python interpreter to find the libs. if(NOT PythonLibsNew_FIND_VERSION) - set(PythonLibsNew_FIND_VERSION "3.6") + set(PythonLibsNew_FIND_VERSION "3.9") +endif() + +if(NOT CMAKE_VERSION VERSION_LESS "3.27") + cmake_policy(GET CMP0148 _pybind11_cmp0148) + if(NOT _pybind11_cmp0148) + message( + AUTHOR_WARNING + "Policy CMP0148 is not set: The FindPythonInterp and FindPythonLibs " + "modules are removed. Run \"cmake --help-policy CMP0148\" for policy " + "details. Use the cmake_policy command to set the policy and suppress " + "this warning, or preferably upgrade to using FindPython, either by " + "calling it explicitly before pybind11, or by setting " + "PYBIND11_FINDPYTHON ON before pybind11.") + endif() + cmake_policy(SET CMP0148 OLD) + unset(_pybind11_cmp0148) endif() find_package(PythonInterp ${PythonLibsNew_FIND_VERSION} ${_pythonlibs_required} @@ -172,17 +188,34 @@ _pybind11_get_if_undef(_PYTHON_VALUES 0 _PYTHON_VERSION_LIST) _pybind11_get_if_undef(_PYTHON_VALUES 1 PYTHON_PREFIX) _pybind11_get_if_undef(_PYTHON_VALUES 2 PYTHON_INCLUDE_DIR) _pybind11_get_if_undef(_PYTHON_VALUES 3 PYTHON_SITE_PACKAGES) -_pybind11_get_if_undef(_PYTHON_VALUES 4 PYTHON_MODULE_EXTENSION) _pybind11_get_if_undef(_PYTHON_VALUES 5 PYTHON_IS_DEBUG) _pybind11_get_if_undef(_PYTHON_VALUES 6 PYTHON_SIZEOF_VOID_P) _pybind11_get_if_undef(_PYTHON_VALUES 7 PYTHON_LIBRARY_SUFFIX) _pybind11_get_if_undef(_PYTHON_VALUES 8 PYTHON_LIBDIR) _pybind11_get_if_undef(_PYTHON_VALUES 9 PYTHON_MULTIARCH) +list(GET _PYTHON_VALUES 4 _PYTHON_MODULE_EXT_SUFFIX) +if(PYBIND11_PYTHONLIBS_OVERWRITE OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX) + get_filename_component(PYTHON_MODULE_DEBUG_POSTFIX "${_PYTHON_MODULE_EXT_SUFFIX}" NAME_WE) +endif() +if(PYBIND11_PYTHONLIBS_OVERWRITE OR NOT DEFINED PYTHON_MODULE_EXTENSION) + get_filename_component(PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXT_SUFFIX}" EXT) + if((NOT "$ENV{SETUPTOOLS_EXT_SUFFIX}" STREQUAL "") AND (NOT "$ENV{SETUPTOOLS_EXT_SUFFIX}" + STREQUAL "${PYTHON_MODULE_EXTENSION}")) + message( + AUTHOR_WARNING, + "SETUPTOOLS_EXT_SUFFIX is set to \"$ENV{SETUPTOOLS_EXT_SUFFIX}\", " + "but the auto-calculated Python extension suffix is \"${PYTHON_MODULE_EXTENSION}\". " + "This may cause problems when importing the Python extensions. " + "If you are using cross-compiling Python, you may need to " + "set PYTHON_MODULE_EXTENSION manually.") + endif() +endif() + # Make sure the Python has the same pointer-size as the chosen compiler # Skip if CMAKE_SIZEOF_VOID_P is not defined # This should be skipped for (non-Apple) cross-compiles (like EMSCRIPTEN) -if(NOT CMAKE_CROSSCOMPILING +if(NOT _PYBIND11_CROSSCOMPILING AND CMAKE_SIZEOF_VOID_P AND (NOT "${PYTHON_SIZEOF_VOID_P}" STREQUAL "${CMAKE_SIZEOF_VOID_P}")) if(PythonLibsNew_FIND_REQUIRED) diff --git a/lib/pybind11/tools/codespell_ignore_lines_from_errors.py b/lib/pybind11/tools/codespell_ignore_lines_from_errors.py index 5403ec3..08ced3d 100644 --- a/lib/pybind11/tools/codespell_ignore_lines_from_errors.py +++ b/lib/pybind11/tools/codespell_ignore_lines_from_errors.py @@ -9,22 +9,27 @@ git diff to review changes, then commit, push. """ +from __future__ import annotations + import sys -from typing import List -def run(args: List[str]) -> None: +def run(args: list[str]) -> None: assert len(args) == 1, "codespell_errors.txt" cache = {} done = set() - for line in sorted(open(args[0]).read().splitlines()): + with open(args[0]) as f: + lines = f.read().splitlines() + + for line in sorted(lines): i = line.find(" ==> ") if i > 0: flds = line[:i].split(":") if len(flds) >= 2: filename, line_num = flds[:2] if filename not in cache: - cache[filename] = open(filename).read().splitlines() + with open(filename) as f: + cache[filename] = f.read().splitlines() supp = cache[filename][int(line_num) - 1] if supp not in done: print(supp) diff --git a/lib/pybind11/tools/libsize.py b/lib/pybind11/tools/libsize.py index 1ac9afb..ef5fb13 100644 --- a/lib/pybind11/tools/libsize.py +++ b/lib/pybind11/tools/libsize.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os import sys diff --git a/lib/pybind11/tools/make_changelog.py b/lib/pybind11/tools/make_changelog.py index b5bd832..8d2c7b0 100755 --- a/lib/pybind11/tools/make_changelog.py +++ b/lib/pybind11/tools/make_changelog.py @@ -1,50 +1,129 @@ -#!/usr/bin/env python3 +#!/usr/bin/env -S uv run +# /// script +# dependencies = ["ghapi>=2", "rich"] +# /// + +from __future__ import annotations + +import os import re +import subprocess import ghapi.all from rich import print from rich.syntax import Syntax -ENTRY = re.compile( +MD_ENTRY = re.compile( r""" - Suggested \s changelog \s entry: - .* - ```rst - \s* - (.*?) - \s* - ``` -""", - re.DOTALL | re.VERBOSE, + ^\#+\ Suggested\ changelog\ entry:?\s*$ # Match the heading, colon optional + (?:\s*)? # Optionally match one comment + (?P.*?) # Lazily capture content until... + (?= # Lookahead for one of the following: + ^-{3,}\s*$ # A line with 3 or more dashes + | ^") + print() + for msg in subs[sub]: + print(Syntax(msg, "md", theme="ansi_light", word_wrap=True)) + print() + print() if missing: print() @@ -56,7 +135,16 @@ print(f"[red] {issue.html_url}\n") print("[bold]Template:\n") - msg = "## Suggested changelog entry:\n\n```rst\n\n```" + msg = "## Suggested changelog entry:" print(Syntax(msg, "md", theme="ansi_light")) +if old: + print() + print("[red]" + "-" * 30) + print() + + for issue in old: + print(f"[red bold]Old:[/red bold][red] {issue.title}") + print(f"[red] {issue.html_url}\n") + print() diff --git a/lib/pybind11/tools/make_global.py b/lib/pybind11/tools/make_global.py new file mode 100755 index 0000000..8d70445 --- /dev/null +++ b/lib/pybind11/tools/make_global.py @@ -0,0 +1,33 @@ +#!/usr/bin/env -S uv run -q + +# /// script +# dependencies = ["tomlkit"] +# /// +from __future__ import annotations + +from pathlib import Path + +import tomlkit + +DIR = Path(__file__).parent.resolve() +PYPROJECT = DIR.parent / "pyproject.toml" + + +def get_global() -> str: + pyproject = tomlkit.parse(PYPROJECT.read_text()) + del pyproject["tool"]["scikit-build"]["generate"] + del pyproject["project"]["entry-points"] + del pyproject["project"]["scripts"] + del pyproject["tool"]["scikit-build"]["metadata"]["optional-dependencies"] + pyproject["project"]["name"] = "pybind11-global" + pyproject["tool"]["scikit-build"]["experimental"] = True + pyproject["tool"]["scikit-build"]["wheel"]["install-dir"] = "/data" + pyproject["tool"]["scikit-build"]["wheel"]["packages"] = [] + + result = tomlkit.dumps(pyproject) + assert isinstance(result, str) + return result + + +if __name__ == "__main__": + print(get_global()) diff --git a/lib/pybind11/tools/pybind11Common.cmake b/lib/pybind11/tools/pybind11Common.cmake index 0c985bc..d75fb67 100644 --- a/lib/pybind11/tools/pybind11Common.cmake +++ b/lib/pybind11/tools/pybind11Common.cmake @@ -2,11 +2,11 @@ Adds the following targets:: - pybind11::pybind11 - link to headers and pybind11 + pybind11::pybind11 - link to Python headers and pybind11::headers pybind11::module - Adds module links pybind11::embed - Adds embed links - pybind11::lto - Link time optimizations (manual selection) - pybind11::thin_lto - Link time optimizations (manual selection) + pybind11::lto - Link time optimizations (only if CMAKE_INTERPROCEDURAL_OPTIMIZATION is not set) + pybind11::thin_lto - Link time optimizations (only if CMAKE_INTERPROCEDURAL_OPTIMIZATION is not set) pybind11::python_link_helper - Adds link to Python libraries pybind11::windows_extras - MSVC bigobj and mp for building multithreaded pybind11::opt_size - avoid optimizations that increase code size @@ -18,12 +18,6 @@ Adds the following functions:: #]======================================================] -# CMake 3.10 has an include_guard command, but we can't use that yet -# include_guard(global) (pre-CMake 3.10) -if(TARGET pybind11::lto) - return() -endif() - # If we are in subdirectory mode, all IMPORTED targets must be GLOBAL. If we # are in CONFIG mode, they should be "normal" targets instead. # In CMake 3.11+ you can promote a target to global after you create it, @@ -32,8 +26,13 @@ get_property( is_config TARGET pybind11::headers PROPERTY IMPORTED) + if(NOT is_config) + include_guard(GLOBAL) set(optional_global GLOBAL) +else() + include_guard(DIRECTORY) + set(optional_global "") endif() # If not run in Python mode, we still would like this to at least @@ -42,6 +41,32 @@ set(pybind11_INCLUDE_DIRS "${pybind11_INCLUDE_DIR}" CACHE INTERNAL "Include directory for pybind11 (Python not requested)") +# CMP0190 prohibits calling FindPython with both Interpreter and Development components +# when cross-compiling, unless the CMAKE_CROSSCOMPILING_EMULATOR variable is defined. +# Default PYBIND11_USE_CROSSCOMPILING to ON in that case, but never override a value the +# project set explicitly (e.g. Emscripten/Pyodide defines an emulator yet still wants it ON). +# PYBIND11_USE_CROSSCOMPILING is undefined for find_package() consumers, but our own +# CMakeLists.txt always defines it via option(); _PYBIND11_USE_CROSSCOMPILING_DEFAULTED tells +# us whether that came from the project or from the option() default. +if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.1" + AND NOT DEFINED CMAKE_CROSSCOMPILING_EMULATOR + AND (NOT DEFINED PYBIND11_USE_CROSSCOMPILING OR _PYBIND11_USE_CROSSCOMPILING_DEFAULTED)) + cmake_policy(GET CMP0190 _pybind11_cmp0190) + if(_pybind11_cmp0190 STREQUAL "NEW") + set(PYBIND11_USE_CROSSCOMPILING "ON") + endif() +endif() + +if(CMAKE_CROSSCOMPILING AND PYBIND11_USE_CROSSCOMPILING) + set(_PYBIND11_CROSSCOMPILING + ON + CACHE INTERNAL "") +else() + set(_PYBIND11_CROSSCOMPILING + OFF + CACHE INTERNAL "") +endif() + # --------------------- Shared targets ---------------------------- # Build an interface library target: @@ -65,27 +90,38 @@ set_property( APPEND PROPERTY INTERFACE_LINK_LIBRARIES pybind11::pybind11) -# --------------------------- link helper --------------------------- - -add_library(pybind11::python_link_helper IMPORTED INTERFACE ${optional_global}) +# -------------- emscripten requires exceptions enabled ------------- +# _pybind11_no_exceptions is a private mechanism to disable this addition. +# Please open an issue if you need to use it; it will be removed if no one +# needs it. +if(CMAKE_SYSTEM_NAME MATCHES Emscripten AND NOT _pybind11_no_exceptions) + if(is_config) + set(_tmp_config_target pybind11::pybind11_headers) + else() + set(_tmp_config_target pybind11_headers) + endif() -if(CMAKE_VERSION VERSION_LESS 3.13) - # In CMake 3.11+, you can set INTERFACE properties via the normal methods, and - # this would be simpler. set_property( - TARGET pybind11::python_link_helper + TARGET ${_tmp_config_target} APPEND - PROPERTY INTERFACE_LINK_LIBRARIES "$<$:-undefined dynamic_lookup>") -else() - # link_options was added in 3.13+ - # This is safer, because you are ensured the deduplication pass in CMake will not consider - # these separate and remove one but not the other. + PROPERTY INTERFACE_LINK_OPTIONS -fexceptions) set_property( - TARGET pybind11::python_link_helper + TARGET ${_tmp_config_target} APPEND - PROPERTY INTERFACE_LINK_OPTIONS "$<$:LINKER:-undefined,dynamic_lookup>") + PROPERTY INTERFACE_COMPILE_OPTIONS -fexceptions) + unset(_tmp_config_target) endif() +# --------------------------- link helper --------------------------- + +add_library(pybind11::python_link_helper IMPORTED INTERFACE ${optional_global}) + +set_property( + TARGET pybind11::python_link_helper + APPEND + PROPERTY INTERFACE_LINK_OPTIONS + "$<$:LINKER:-undefined,dynamic_lookup>") + # ------------------------ Windows extras ------------------------- add_library(pybind11::windows_extras IMPORTED INTERFACE ${optional_global}) @@ -100,22 +136,14 @@ if(MSVC) # That's also clang-cl # /MP enables multithreaded builds (relevant when there are many files) for MSVC if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") # no Clang no Intel - if(CMAKE_VERSION VERSION_LESS 3.11) - set_property( - TARGET pybind11::windows_extras - APPEND - PROPERTY INTERFACE_COMPILE_OPTIONS $<$>:/MP>) - else() - # Only set these options for C++ files. This is important so that, for - # instance, projects that include other types of source files like CUDA - # .cu files don't get these options propagated to nvcc since that would - # cause the build to fail. - set_property( - TARGET pybind11::windows_extras - APPEND - PROPERTY INTERFACE_COMPILE_OPTIONS - $<$>:$<$:/MP>>) - endif() + # Only set these options for C++ files. This is important so that, for + # instance, projects that include other types of source files like CUDA + # .cu files don't get these options propagated to nvcc since that would + # cause the build to fail. + set_property( + TARGET pybind11::windows_extras + APPEND + PROPERTY INTERFACE_COMPILE_OPTIONS $<$>:$<$:/MP>>) endif() endif() @@ -163,16 +191,51 @@ endif() # --------------------- Python specifics ------------------------- +# CMake 3.27 removes the classic FindPythonInterp if CMP0148 is NEW +if(CMAKE_VERSION VERSION_LESS "3.27") + set(_pybind11_missing_old_python "OLD") +else() + cmake_policy(GET CMP0148 _pybind11_missing_old_python) +endif() + # Check to see which Python mode we are in, new, old, or no python if(PYBIND11_NOPYTHON) set(_pybind11_nopython ON) + # We won't use new FindPython if PYBIND11_FINDPYTHON is defined and falselike + # Otherwise, we use if FindPythonLibs is missing or if FindPython was already used elseif( - PYBIND11_FINDPYTHON - OR Python_FOUND - OR Python2_FOUND - OR Python3_FOUND) + (NOT DEFINED PYBIND11_FINDPYTHON + OR PYBIND11_FINDPYTHON STREQUAL "COMPAT" + OR PYBIND11_FINDPYTHON) + AND (_pybind11_missing_old_python STREQUAL "NEW" + OR PYBIND11_FINDPYTHON STREQUAL "COMPAT" + OR PYBIND11_FINDPYTHON + OR Python_FOUND + OR Python3_FOUND + )) + # New mode - include("${CMAKE_CURRENT_LIST_DIR}/pybind11NewTools.cmake") + if(Python_FOUND OR Python3_FOUND) + include("${CMAKE_CURRENT_LIST_DIR}/pybind11NewTools.cmake") + else() + include("${CMAKE_CURRENT_LIST_DIR}/pybind11NewTools.cmake") + + if(PYBIND11_FINDPYTHON STREQUAL "COMPAT") + message( + "Using compatibility mode for Python, set PYBIND11_FINDPYTHON to NEW/OLD to silence this message" + ) + set(PYTHON_EXECUTABLE "${Python_EXECUTABLE}") + set(PYTHON_INCLUDE_DIR "${Python_INCLUDE_DIR}") + set(Python_INCLUDE_DIRS "${Python_INCLUDE_DIRS}") + set(PYTHON_LIBRARY "${Python_LIBRARY}") + set(PYTHON_LIBRARIES "${Python_LIBRARIES}") + set(PYTHON_VERSION "${Python_VERSION}") + set(PYTHON_VERSION_STRING "${Python_VERSION_STRING}") + set(PYTHON_VERSION_MAJOR "${Python_VERSION_MAJOR}") + set(PYTHON_VERSION_MINOR "${Python_VERSION_MINOR}") + set(PYTHON_VERSION_PATCH "${Python_VERSION_PATCH}") + endif() + endif() else() @@ -183,7 +246,7 @@ endif() # --------------------- pybind11_find_import ------------------------------- -if(NOT _pybind11_nopython) +if(NOT _pybind11_nopython AND NOT _PYBIND11_CROSSCOMPILING) # Check to see if modules are importable. Use REQUIRED to force an error if # one of the modules is not found. _FOUND will be set if the # package was found (underscores replace dashes if present). QUIET will hide @@ -210,8 +273,15 @@ if(NOT _pybind11_nopython) execute_process( COMMAND - ${${_Python}_EXECUTABLE} -c - "from pkg_resources import get_distribution; print(get_distribution('${PYPI_NAME}').version)" + ${${_Python}_EXECUTABLE} -c " +try: + from importlib.metadata import version +except ImportError: + from pkg_resources import get_distribution + def version(s): + return get_distribution(s).version +print(version('${PYPI_NAME}')) + " RESULT_VARIABLE RESULT_PRESENT OUTPUT_VARIABLE PKG_VERSION ERROR_QUIET) @@ -288,25 +358,37 @@ function(_pybind11_generate_lto target prefer_thin_lto) if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND NOT APPLE) # Clang Gold plugin does not support -Os; append -O3 to MinSizeRel builds to override it set(linker_append ";$<$:-O3>") - elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND NOT MINGW) + elseif( + CMAKE_CXX_COMPILER_ID MATCHES "GNU" + AND NOT MINGW + AND NOT APPLE) + # GCC on macOS has no linker plugin, which -fno-fat-lto-objects requires set(cxx_append ";-fno-fat-lto-objects") endif() - if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le" OR CMAKE_SYSTEM_PROCESSOR MATCHES "mips64") - set(NO_FLTO_ARCH TRUE) + if(prefer_thin_lto) + set(thin "=thin") else() - set(NO_FLTO_ARCH FALSE) + set(thin "") endif() - if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" - AND prefer_thin_lto - AND NOT NO_FLTO_ARCH) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le" OR CMAKE_SYSTEM_PROCESSOR MATCHES "mips64") + # Do nothing + elseif(CMAKE_SYSTEM_NAME MATCHES Emscripten) + # This compile is very costly when cross-compiling, so set this without checking + set(PYBIND11_LTO_CXX_FLAGS "-flto${thin}${cxx_append}") + set(PYBIND11_LTO_LINKER_FLAGS "-flto${thin}${linker_append}") + elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang") _pybind11_return_if_cxx_and_linker_flags_work( - HAS_FLTO_THIN "-flto=thin${cxx_append}" "-flto=thin${linker_append}" + HAS_FLTO_THIN "-flto${thin}${cxx_append}" "-flto${thin}${linker_append}" PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) endif() - - if(NOT HAS_FLTO_THIN AND NOT NO_FLTO_ARCH) + if(NOT HAS_FLTO_THIN) + _pybind11_return_if_cxx_and_linker_flags_work( + HAS_FLTO_AUTO "-flto=auto${cxx_append}" "-flto=auto${linker_append}" + PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) + endif() + if(NOT HAS_FLTO_AUTO) _pybind11_return_if_cxx_and_linker_flags_work( HAS_FLTO "-flto${cxx_append}" "-flto${linker_append}" PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) @@ -339,11 +421,7 @@ function(_pybind11_generate_lto target prefer_thin_lto) set(is_debug "$,$>") set(not_debug "$") set(cxx_lang "$") - if(MSVC AND CMAKE_VERSION VERSION_LESS 3.11) - set(genex "${not_debug}") - else() - set(genex "$") - endif() + set(genex "$") set_property( TARGET ${target} APPEND @@ -358,25 +436,20 @@ function(_pybind11_generate_lto target prefer_thin_lto) endif() if(PYBIND11_LTO_LINKER_FLAGS) - if(CMAKE_VERSION VERSION_LESS 3.11) - set_property( - TARGET ${target} - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES "$<${not_debug}:${PYBIND11_LTO_LINKER_FLAGS}>") - else() - set_property( - TARGET ${target} - APPEND - PROPERTY INTERFACE_LINK_OPTIONS "$<${not_debug}:${PYBIND11_LTO_LINKER_FLAGS}>") - endif() + set_property( + TARGET ${target} + APPEND + PROPERTY INTERFACE_LINK_OPTIONS "$<${not_debug}:${PYBIND11_LTO_LINKER_FLAGS}>") endif() endfunction() -add_library(pybind11::lto IMPORTED INTERFACE ${optional_global}) -_pybind11_generate_lto(pybind11::lto FALSE) +if(NOT DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION) + add_library(pybind11::lto IMPORTED INTERFACE ${optional_global}) + _pybind11_generate_lto(pybind11::lto FALSE) -add_library(pybind11::thin_lto IMPORTED INTERFACE ${optional_global}) -_pybind11_generate_lto(pybind11::thin_lto TRUE) + add_library(pybind11::thin_lto IMPORTED INTERFACE ${optional_global}) + _pybind11_generate_lto(pybind11::thin_lto TRUE) +endif() # ---------------------- pybind11_strip ----------------------------- diff --git a/lib/pybind11/tools/pybind11Config.cmake.in b/lib/pybind11/tools/pybind11Config.cmake.in index 9383e8c..f52b2fb 100644 --- a/lib/pybind11/tools/pybind11Config.cmake.in +++ b/lib/pybind11/tools/pybind11Config.cmake.in @@ -63,7 +63,16 @@ Modes There are two modes provided; classic, which is built on the old Python discovery packages in CMake, or the new FindPython mode, which uses FindPython -from 3.12+ forward (3.15+ _highly_ recommended). +from 3.12+ forward (3.15+ _highly_ recommended). If you set the minimum or +maximum version of CMake to 3.27+, then FindPython is the default (since +FindPythonInterp/FindPythonLibs has been removed via policy `CMP0148`). + +Starting in pybind11 3.0, the new mode is the default, but we provide backward +compatible names (``PYTHON_*`` vs. the new ``Python_*`` names). Set the mode +explicitly to avoid the compatibility defines. You can specify this mode +explicitly by setting ``PYBIND11_FINDPYTHON`` to ``COMPAT``, but if you are changing +your CMakeLists anyway, please just use the ``ON`` mode. A future release may +default to ``ON``. New FindPython mode ^^^^^^^^^^^^^^^^^^^ @@ -82,10 +91,10 @@ you can either use the basic targets, or use the FindPython tools: # Python method: Python_add_library(MyModule2 src2.cpp) - target_link_libraries(MyModule2 pybind11::headers) + target_link_libraries(MyModule2 PUBLIC pybind11::headers) set_target_properties(MyModule2 PROPERTIES INTERPROCEDURAL_OPTIMIZATION ON - CXX_VISIBILITY_PRESET ON + CXX_VISIBILITY_PRESET hidden VISIBILITY_INLINES_HIDDEN ON) If you build targets yourself, you may be interested in stripping the output @@ -147,10 +156,10 @@ default is ``MODULE``. There are several options: ``OPT_SIZE`` Optimize for size, even if the ``CMAKE_BUILD_TYPE`` is not ``MinSizeRel``. ``THIN_LTO`` - Use thin TLO instead of regular if there's a choice (pybind11's selection + Use thin LTO instead of regular if there's a choice (pybind11's selection is disabled if ``CMAKE_INTERPROCEDURAL_OPTIMIZATIONS`` is set). ``WITHOUT_SOABI`` - Disable the SOABI component (``PYBIND11_NEWPYTHON`` mode only). + Disable the SOABI component (``PYBIND11_FINDPYTHON`` mode only). ``NO_EXTRAS`` Disable all extras, exit immediately after making the module. diff --git a/lib/pybind11/tools/pybind11GuessPythonExtSuffix.cmake b/lib/pybind11/tools/pybind11GuessPythonExtSuffix.cmake new file mode 100644 index 0000000..f145739 --- /dev/null +++ b/lib/pybind11/tools/pybind11GuessPythonExtSuffix.cmake @@ -0,0 +1,94 @@ +cmake_minimum_required(VERSION 3.15...4.2) + +function(pybind11_guess_python_module_extension python) + + # The SETUPTOOLS_EXT_SUFFIX environment variable takes precedence: + if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX AND DEFINED ENV{SETUPTOOLS_EXT_SUFFIX}) + message( + STATUS + "Getting Python extension suffix from ENV{SETUPTOOLS_EXT_SUFFIX}: $ENV{SETUPTOOLS_EXT_SUFFIX}" + ) + set(PYTHON_MODULE_EXT_SUFFIX + "$ENV{SETUPTOOLS_EXT_SUFFIX}" + CACHE + STRING + "Extension suffix for Python extension modules (Initialized from SETUPTOOLS_EXT_SUFFIX)") + endif() + + # The final extension depends on the system + set(_PY_BUILD_EXTENSION "${CMAKE_SHARED_MODULE_SUFFIX}") + if(CMAKE_SYSTEM_NAME STREQUAL "Windows") + set(_PY_BUILD_EXTENSION ".pyd") + endif() + + # If running under scikit-build-core, use the SKBUILD_SOABI variable: + if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX AND DEFINED SKBUILD_SOABI) + message(STATUS "Determining Python extension suffix based on SKBUILD_SOABI: ${SKBUILD_SOABI}") + set(PYTHON_MODULE_EXT_SUFFIX ".${SKBUILD_SOABI}${_PY_BUILD_EXTENSION}") + endif() + + # If that didn't work, use the Python_SOABI variable: + if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX AND DEFINED ${python}_SOABI) + message( + STATUS "Determining Python extension suffix based on ${python}_SOABI: ${${python}_SOABI}") + # If the SOABI already has an extension, use it as the full suffix + # (used for debug versions of Python on Windows) + if(${python}_SOABI MATCHES "\\.") + set(PYTHON_MODULE_EXT_SUFFIX "${${python}_SOABI}") + # If the SOABI is empty, this is usually a bug, but we generate a + # correct extension anyway, which is the best we can do + elseif("${${python}_SOABI}" STREQUAL "") + message( + WARNING + "${python}_SOABI is defined but empty. You may want to set PYTHON_MODULE_EXT_SUFFIX explicitly." + ) + set(PYTHON_MODULE_EXT_SUFFIX "${_PY_BUILD_EXTENSION}") + # Otherwise, add the system-dependent extension to it + else() + set(PYTHON_MODULE_EXT_SUFFIX ".${${python}_SOABI}${_PY_BUILD_EXTENSION}") + endif() + endif() + + # If we could not deduce the extension suffix, unset the results: + if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX) + unset(PYTHON_MODULE_DEBUG_POSTFIX CACHE) + unset(PYTHON_MODULE_EXTENSION CACHE) + unset(PYTHON_IS_DEBUG CACHE) + return() + endif() + + # Sanity checks: + if(${python}_SOABI AND NOT (PYTHON_MODULE_EXT_SUFFIX STREQUAL ${python}_SOABI + OR PYTHON_MODULE_EXT_SUFFIX MATCHES "\\.${${python}_SOABI}\\.")) + message( + WARNING + "Python extension suffix (${PYTHON_MODULE_EXT_SUFFIX}) does not match ${python}_SOABI (${${python}_SOABI})." + ) + endif() + + # Separate file name postfix from extension: (https://github.com/pybind/pybind11/issues/4699) + get_filename_component(_PYTHON_MODULE_DEBUG_POSTFIX "${PYTHON_MODULE_EXT_SUFFIX}" NAME_WE) + get_filename_component(_PYTHON_MODULE_EXTENSION "${PYTHON_MODULE_EXT_SUFFIX}" EXT) + + # Try to deduce the debug ABI from the extension suffix: + if(NOT DEFINED _PYTHON_IS_DEBUG) + if(_PYTHON_MODULE_EXTENSION MATCHES "^\\.(cpython-|cp|pypy)[0-9]+dm?-" + OR _PYTHON_MODULE_DEBUG_POSTFIX MATCHES "^_d") + set(_PYTHON_IS_DEBUG On) + else() + set(_PYTHON_IS_DEBUG Off) + endif() + endif() + + # Return results + set(PYTHON_MODULE_DEBUG_POSTFIX + "${_PYTHON_MODULE_DEBUG_POSTFIX}" + CACHE INTERNAL "") + set(PYTHON_MODULE_EXTENSION + "${_PYTHON_MODULE_EXTENSION}" + CACHE INTERNAL "") + set(PYTHON_IS_DEBUG + "${_PYTHON_IS_DEBUG}" + CACHE INTERNAL "") + +endfunction() diff --git a/lib/pybind11/tools/pybind11NewTools.cmake b/lib/pybind11/tools/pybind11NewTools.cmake index 7d7424a..b0fe207 100644 --- a/lib/pybind11/tools/pybind11NewTools.cmake +++ b/lib/pybind11/tools/pybind11NewTools.cmake @@ -5,10 +5,6 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. -if(CMAKE_VERSION VERSION_LESS 3.12) - message(FATAL_ERROR "You cannot use the new FindPython module with CMake < 3.12") -endif() - include_guard(DIRECTORY) get_property( @@ -32,17 +28,64 @@ if(NOT Python_FOUND AND NOT Python3_FOUND) set(Python_ROOT_DIR "$ENV{pythonLocation}") endif() - find_package(Python 3.6 REQUIRED COMPONENTS Interpreter Development ${_pybind11_quiet}) + # Interpreter should not be found when cross-compiling + if(_PYBIND11_CROSSCOMPILING) + set(_pybind11_interp_component "") + else() + set(_pybind11_interp_component Interpreter) + endif() + + # Development.Module support (required for manylinux) started in 3.18 + if(CMAKE_VERSION VERSION_LESS 3.18) + set(_pybind11_dev_component Development) + else() + set(_pybind11_dev_component Development.Module OPTIONAL_COMPONENTS Development.Embed) + endif() + + # Callers need to be able to access Python_EXECUTABLE + set(_pybind11_global_keyword "") + if(NOT is_config AND NOT DEFINED Python_ARTIFACTS_INTERACTIVE) + set(Python_ARTIFACTS_INTERACTIVE TRUE) + if(NOT CMAKE_VERSION VERSION_LESS 3.24) + set(_pybind11_global_keyword "GLOBAL") + endif() + endif() + + find_package( + Python 3.9 REQUIRED COMPONENTS ${_pybind11_interp_component} ${_pybind11_dev_component} + ${_pybind11_quiet} ${_pybind11_global_keyword}) # If we are in submodule mode, export the Python targets to global targets. # If this behavior is not desired, FindPython _before_ pybind11. - if(NOT is_config) - set_property(TARGET Python::Python PROPERTY IMPORTED_GLOBAL TRUE) - set_property(TARGET Python::Interpreter PROPERTY IMPORTED_GLOBAL TRUE) + if(NOT is_config + AND Python_ARTIFACTS_INTERACTIVE + AND _pybind11_global_keyword STREQUAL "") + if(TARGET Python::Python) + set_property(TARGET Python::Python PROPERTY IMPORTED_GLOBAL TRUE) + endif() + if(TARGET Python::Interpreter) + set_property(TARGET Python::Interpreter PROPERTY IMPORTED_GLOBAL TRUE) + endif() if(TARGET Python::Module) set_property(TARGET Python::Module PROPERTY IMPORTED_GLOBAL TRUE) endif() endif() + + # Explicitly export version for callers (including our own functions) + if(NOT is_config AND Python_ARTIFACTS_INTERACTIVE) + set(Python_VERSION + "${Python_VERSION}" + CACHE INTERNAL "") + set(Python_VERSION_MAJOR + "${Python_VERSION_MAJOR}" + CACHE INTERNAL "") + set(Python_VERSION_MINOR + "${Python_VERSION_MINOR}" + CACHE INTERNAL "") + set(Python_VERSION_PATCH + "${Python_VERSION_PATCH}" + CACHE INTERNAL "") + endif() endif() if(Python_FOUND) @@ -63,61 +106,94 @@ if(PYBIND11_MASTER_PROJECT) endif() endif() -# If a user finds Python, they may forget to include the Interpreter component -# and the following two steps require it. It is highly recommended by CMake -# when finding development libraries anyway, so we will require it. -if(NOT DEFINED ${_Python}_EXECUTABLE) - message( - FATAL_ERROR - "${_Python} was found without the Interpreter component. Pybind11 requires this component.") - -endif() +if(NOT _PYBIND11_CROSSCOMPILING AND DEFINED ${_Python}_EXECUTABLE) + if(DEFINED PYBIND11_PYTHON_EXECUTABLE_LAST AND NOT ${_Python}_EXECUTABLE STREQUAL + PYBIND11_PYTHON_EXECUTABLE_LAST) + # Detect changes to the Python version/binary in subsequent CMake runs, and refresh config if needed + unset(PYTHON_IS_DEBUG CACHE) + unset(PYTHON_MODULE_EXTENSION CACHE) + unset(PYTHON_MODULE_DEBUG_POSTFIX CACHE) + endif() -if(NOT ${_Python}_EXECUTABLE STREQUAL PYBIND11_PYTHON_EXECUTABLE_LAST) - # Detect changes to the Python version/binary in subsequent CMake runs, and refresh config if needed - unset(PYTHON_IS_DEBUG CACHE) - unset(PYTHON_MODULE_EXTENSION CACHE) set(PYBIND11_PYTHON_EXECUTABLE_LAST "${${_Python}_EXECUTABLE}" CACHE INTERNAL "Python executable during the last CMake run") -endif() -if(NOT DEFINED PYTHON_IS_DEBUG) - # Debug check - see https://stackoverflow.com/questions/646518/python-how-to-detect-debug-Interpreter - execute_process( - COMMAND "${${_Python}_EXECUTABLE}" "-c" - "import sys; sys.exit(hasattr(sys, 'gettotalrefcount'))" - RESULT_VARIABLE _PYTHON_IS_DEBUG) - set(PYTHON_IS_DEBUG - "${_PYTHON_IS_DEBUG}" - CACHE INTERNAL "Python debug status") -endif() + if(NOT DEFINED PYTHON_IS_DEBUG) + # Debug check - see https://stackoverflow.com/questions/646518/python-how-to-detect-debug-Interpreter + execute_process( + COMMAND "${${_Python}_EXECUTABLE}" "-c" + "import sys; sys.exit(hasattr(sys, 'gettotalrefcount'))" + RESULT_VARIABLE _PYTHON_IS_DEBUG) + set(PYTHON_IS_DEBUG + "${_PYTHON_IS_DEBUG}" + CACHE INTERNAL "Python debug status") + endif() + + # Get the suffix - SO is deprecated, should use EXT_SUFFIX, but this is + # required for PyPy3 (as of 7.3.1) + if(NOT DEFINED PYTHON_MODULE_EXTENSION OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX) + execute_process( + COMMAND + "${${_Python}_EXECUTABLE}" "-c" + "import sys, importlib; s = importlib.import_module('distutils.sysconfig' if sys.version_info < (3, 10) else 'sysconfig'); print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO'))" + OUTPUT_VARIABLE _PYTHON_MODULE_EXT_SUFFIX + ERROR_VARIABLE _PYTHON_MODULE_EXT_SUFFIX_ERR + OUTPUT_STRIP_TRAILING_WHITESPACE) + + if(_PYTHON_MODULE_EXT_SUFFIX STREQUAL "") + message( + FATAL_ERROR + "pybind11 could not query the module file extension, likely the 'distutils'" + "package is not installed. Full error message:\n${_PYTHON_MODULE_EXT_SUFFIX_ERR}") + endif() + + # This needs to be available for the pybind11_extension function + if(NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX) + get_filename_component(_PYTHON_MODULE_DEBUG_POSTFIX "${_PYTHON_MODULE_EXT_SUFFIX}" NAME_WE) + set(PYTHON_MODULE_DEBUG_POSTFIX + "${_PYTHON_MODULE_DEBUG_POSTFIX}" + CACHE INTERNAL "") + endif() -# Get the suffix - SO is deprecated, should use EXT_SUFFIX, but this is -# required for PyPy3 (as of 7.3.1) -if(NOT DEFINED PYTHON_MODULE_EXTENSION) - execute_process( - COMMAND - "${${_Python}_EXECUTABLE}" "-c" - "import sys, importlib; s = importlib.import_module('distutils.sysconfig' if sys.version_info < (3, 10) else 'sysconfig'); print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO'))" - OUTPUT_VARIABLE _PYTHON_MODULE_EXTENSION - ERROR_VARIABLE _PYTHON_MODULE_EXTENSION_ERR - OUTPUT_STRIP_TRAILING_WHITESPACE) - - if(_PYTHON_MODULE_EXTENSION STREQUAL "") + if(NOT DEFINED PYTHON_MODULE_EXTENSION) + get_filename_component(_PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXT_SUFFIX}" EXT) + set(PYTHON_MODULE_EXTENSION + "${_PYTHON_MODULE_EXTENSION}" + CACHE INTERNAL "") + if((NOT "$ENV{SETUPTOOLS_EXT_SUFFIX}" STREQUAL "") + AND (NOT "$ENV{SETUPTOOLS_EXT_SUFFIX}" STREQUAL "${PYTHON_MODULE_EXTENSION}")) + message( + AUTHOR_WARNING, + "SETUPTOOLS_EXT_SUFFIX is set to \"$ENV{SETUPTOOLS_EXT_SUFFIX}\", " + "but the auto-calculated Python extension suffix is \"${PYTHON_MODULE_EXTENSION}\". " + "This may cause problems when importing the Python extensions. " + "If you are using cross-compiling Python, you may need to " + "set PYTHON_MODULE_EXTENSION manually.") + endif() + endif() + endif() +else() + if(NOT DEFINED PYTHON_IS_DEBUG + OR NOT DEFINED PYTHON_MODULE_EXTENSION + OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX) + include("${CMAKE_CURRENT_LIST_DIR}/pybind11GuessPythonExtSuffix.cmake") + pybind11_guess_python_module_extension("${_Python}") + endif() + if(NOT DEFINED PYTHON_IS_DEBUG + OR NOT DEFINED PYTHON_MODULE_EXTENSION + OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX) message( - FATAL_ERROR "pybind11 could not query the module file extension, likely the 'distutils'" - "package is not installed. Full error message:\n${_PYTHON_MODULE_EXTENSION_ERR}") + FATAL_ERROR + "A Python interpreter was not found, or you are cross-compiling, and the " + "PYTHON_IS_DEBUG, PYTHON_MODULE_EXTENSION and PYTHON_MODULE_DEBUG_POSTFIX " + "variables could not be guessed. Set these variables appropriately before " + "loading pybind11 (e.g. in your CMake toolchain file)") endif() - - # This needs to be available for the pybind11_extension function - set(PYTHON_MODULE_EXTENSION - "${_PYTHON_MODULE_EXTENSION}" - CACHE INTERNAL "") endif() -# Python debug libraries expose slightly different objects before 3.8 -# https://docs.python.org/3.6/c-api/intro.html#debugging-builds +# A debug build of Python needs Py_DEBUG to select the matching ABI. +# https://docs.python.org/3/c-api/intro.html#debugging-builds # https://stackoverflow.com/questions/39161202/how-to-work-around-missing-pymodule-create2-in-amd64-win-python35-d-lib if(PYTHON_IS_DEBUG) set_property( @@ -156,8 +232,15 @@ if(TARGET ${_Python}::Python) PROPERTY INTERFACE_LINK_LIBRARIES ${_Python}::Python) endif() -# CMake 3.15+ has this if(TARGET ${_Python}::Module) + # On Android, older versions of CMake don't know that modules need to link against + # libpython, so Python::Module will be an INTERFACE target with no associated library + # files. + get_target_property(module_target_type ${_Python}::Module TYPE) + if(ANDROID AND module_target_type STREQUAL INTERFACE_LIBRARY) + target_link_libraries(${_Python}::Module INTERFACE ${${_Python}_LIBRARIES}) + endif() + set_property( TARGET pybind11::module APPEND @@ -199,10 +282,6 @@ function(pybind11_add_module target_name) target_link_libraries(${target_name} PRIVATE pybind11::embed) endif() - if(MSVC) - target_link_libraries(${target_name} PRIVATE pybind11::windows_extras) - endif() - # -fvisibility=hidden is required to allow multiple modules compiled against # different pybind versions to work properly, and for some features (e.g. # py::module_local). We force it on everything inside the `pybind11` @@ -233,11 +312,13 @@ function(pybind11_add_module target_name) endif() endif() - # Use case-insensitive comparison to match the result of $ - string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) - if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO) - # Strip unnecessary sections of the binary on Linux/macOS - pybind11_strip(${target_name}) + if(DEFINED CMAKE_BUILD_TYPE) # see https://github.com/pybind/pybind11/issues/4454 + # Use case-insensitive comparison to match the result of $ + string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) + if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO|NONE) + # Strip unnecessary sections of the binary on Linux/macOS + pybind11_strip(${target_name}) + endif() endif() if(MSVC) @@ -251,6 +332,9 @@ endfunction() function(pybind11_extension name) # The extension is precomputed - set_target_properties(${name} PROPERTIES PREFIX "" SUFFIX "${PYTHON_MODULE_EXTENSION}") - + set_target_properties( + ${name} + PROPERTIES PREFIX "" + DEBUG_POSTFIX "${PYTHON_MODULE_DEBUG_POSTFIX}" + SUFFIX "${PYTHON_MODULE_EXTENSION}") endfunction() diff --git a/lib/pybind11/tools/pybind11Tools.cmake b/lib/pybind11/tools/pybind11Tools.cmake index 66ad00a..81faee7 100644 --- a/lib/pybind11/tools/pybind11Tools.cmake +++ b/lib/pybind11/tools/pybind11Tools.cmake @@ -5,13 +5,7 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. -# include_guard(global) (pre-CMake 3.10) -if(TARGET pybind11::python_headers) - return() -endif() - -# Built-in in CMake 3.5+ -include(CMakeParseArguments) +include_guard(DIRECTORY) if(pybind11_FIND_QUIETLY) set(_pybind11_quiet QUIET) @@ -43,7 +37,7 @@ endif() # A user can set versions manually too set(Python_ADDITIONAL_VERSIONS - "3.11;3.10;3.9;3.8;3.7;3.6" + "3.15;3.14;3.13;3.12;3.11;3.10;3.9" CACHE INTERNAL "") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") @@ -65,6 +59,7 @@ _pybind11_promote_to_cache(PYTHON_INCLUDE_DIRS) _pybind11_promote_to_cache(PYTHON_LIBRARIES) _pybind11_promote_to_cache(PYTHON_MODULE_PREFIX) _pybind11_promote_to_cache(PYTHON_MODULE_EXTENSION) +_pybind11_promote_to_cache(PYTHON_MODULE_DEBUG_POSTFIX) _pybind11_promote_to_cache(PYTHON_VERSION_MAJOR) _pybind11_promote_to_cache(PYTHON_VERSION_MINOR) _pybind11_promote_to_cache(PYTHON_VERSION) @@ -105,8 +100,8 @@ set(pybind11_INCLUDE_DIRS "${pybind11_INCLUDE_DIR}" "${PYTHON_INCLUDE_DIRS}" CACHE INTERNAL "Directories where pybind11 and possibly Python headers are located") -# Python debug libraries expose slightly different objects before 3.8 -# https://docs.python.org/3.6/c-api/intro.html#debugging-builds +# A debug build of Python needs Py_DEBUG to select the matching ABI. +# https://docs.python.org/3/c-api/intro.html#debugging-builds # https://stackoverflow.com/questions/39161202/how-to-work-around-missing-pymodule-create2-in-amd64-win-python35-d-lib if(PYTHON_IS_DEBUG) set_property( @@ -115,41 +110,28 @@ if(PYTHON_IS_DEBUG) PROPERTY INTERFACE_COMPILE_DEFINITIONS Py_DEBUG) endif() -# The <3.11 code here does not support release/debug builds at the same time, like on vcpkg -if(CMAKE_VERSION VERSION_LESS 3.11) - set_property( - TARGET pybind11::module - APPEND - PROPERTY - INTERFACE_LINK_LIBRARIES - pybind11::python_link_helper - "$<$,$>:$>" - ) - - set_property( - TARGET pybind11::embed - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES pybind11::pybind11 $) -else() - # The IMPORTED INTERFACE library here is to ensure that "debug" and "release" get processed outside - # of a generator expression - https://gitlab.kitware.com/cmake/cmake/-/issues/18424, as they are - # target_link_library keywords rather than real libraries. - add_library(pybind11::_ClassicPythonLibraries IMPORTED INTERFACE) - target_link_libraries(pybind11::_ClassicPythonLibraries INTERFACE ${PYTHON_LIBRARIES}) - target_link_libraries( - pybind11::module - INTERFACE - pybind11::python_link_helper - "$<$,$>:pybind11::_ClassicPythonLibraries>") - - target_link_libraries(pybind11::embed INTERFACE pybind11::pybind11 - pybind11::_ClassicPythonLibraries) -endif() +# The IMPORTED INTERFACE library here is to ensure that "debug" and "release" get processed outside +# of a generator expression - https://gitlab.kitware.com/cmake/cmake/-/issues/18424, as they are +# target_link_library keywords rather than real libraries. +add_library(pybind11::_ClassicPythonLibraries IMPORTED INTERFACE) +target_link_libraries(pybind11::_ClassicPythonLibraries INTERFACE ${PYTHON_LIBRARIES}) +target_link_libraries( + pybind11::module + INTERFACE + pybind11::python_link_helper + "$<$,$,$>:pybind11::_ClassicPythonLibraries>" +) + +target_link_libraries(pybind11::embed INTERFACE pybind11::pybind11 + pybind11::_ClassicPythonLibraries) function(pybind11_extension name) # The prefix and extension are provided by FindPythonLibsNew.cmake - set_target_properties(${name} PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}" - SUFFIX "${PYTHON_MODULE_EXTENSION}") + set_target_properties( + ${name} + PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}" + DEBUG_POSTFIX "${PYTHON_MODULE_DEBUG_POSTFIX}" + SUFFIX "${PYTHON_MODULE_EXTENSION}") endfunction() # Build a Python extension module: @@ -212,10 +194,12 @@ function(pybind11_add_module target_name) endif() endif() - # Use case-insensitive comparison to match the result of $ - string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) - if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO) - pybind11_strip(${target_name}) + if(DEFINED CMAKE_BUILD_TYPE) # see https://github.com/pybind/pybind11/issues/4454 + # Use case-insensitive comparison to match the result of $ + string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) + if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO|NONE) + pybind11_strip(${target_name}) + endif() endif() if(MSVC) diff --git a/lib/pybind11/tools/pyproject.toml b/lib/pybind11/tools/pyproject.toml deleted file mode 100644 index 8fe2f47..0000000 --- a/lib/pybind11/tools/pyproject.toml +++ /dev/null @@ -1,3 +0,0 @@ -[build-system] -requires = ["setuptools>=42", "wheel"] -build-backend = "setuptools.build_meta" diff --git a/lib/pybind11/tools/setup_global.py.in b/lib/pybind11/tools/setup_global.py.in deleted file mode 100644 index 885ac5c..0000000 --- a/lib/pybind11/tools/setup_global.py.in +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python3 - -# Setup script for pybind11-global (in the sdist or in tools/setup_global.py in the repository) -# This package is targeted for easy use from CMake. - -import glob -import os -import re - -# Setuptools has to be before distutils -from setuptools import setup - -from distutils.command.install_headers import install_headers - -class InstallHeadersNested(install_headers): - def run(self): - headers = self.distribution.headers or [] - for header in headers: - # Remove pybind11/include/ - short_header = header.split("/", 2)[-1] - - dst = os.path.join(self.install_dir, os.path.dirname(short_header)) - self.mkpath(dst) - (out, _) = self.copy_file(header, dst) - self.outfiles.append(out) - - -main_headers = glob.glob("pybind11/include/pybind11/*.h") -detail_headers = glob.glob("pybind11/include/pybind11/detail/*.h") -eigen_headers = glob.glob("pybind11/include/pybind11/eigen/*.h") -stl_headers = glob.glob("pybind11/include/pybind11/stl/*.h") -cmake_files = glob.glob("pybind11/share/cmake/pybind11/*.cmake") -pkgconfig_files = glob.glob("pybind11/share/pkgconfig/*.pc") -headers = main_headers + detail_headers + stl_headers + eigen_headers - -cmdclass = {"install_headers": InstallHeadersNested} -$extra_cmd - -# This will _not_ affect installing from wheels, -# only building wheels or installing from SDist. -# Primarily intended on Windows, where this is sometimes -# customized (for example, conda-forge uses Library/) -base = os.environ.get("PYBIND11_GLOBAL_PREFIX", "") - -# Must have a separator -if base and not base.endswith("/"): - base += "/" - -setup( - name="pybind11_global", - version="$version", - packages=[], - headers=headers, - data_files=[ - (base + "share/cmake/pybind11", cmake_files), - (base + "share/pkgconfig", pkgconfig_files), - (base + "include/pybind11", main_headers), - (base + "include/pybind11/detail", detail_headers), - (base + "include/pybind11/eigen", eigen_headers), - (base + "include/pybind11/stl", stl_headers), - ], - cmdclass=cmdclass, -) diff --git a/lib/pybind11/tools/setup_main.py.in b/lib/pybind11/tools/setup_main.py.in deleted file mode 100644 index 6358cc7..0000000 --- a/lib/pybind11/tools/setup_main.py.in +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python3 - -# Setup script (in the sdist or in tools/setup_main.py in the repository) - -from setuptools import setup - -cmdclass = {} -$extra_cmd - -setup( - name="pybind11", - version="$version", - download_url='https://github.com/pybind/pybind11/tarball/v$version', - packages=[ - "pybind11", - "pybind11.include.pybind11", - "pybind11.include.pybind11.detail", - "pybind11.include.pybind11.eigen", - "pybind11.include.pybind11.stl", - "pybind11.share.cmake.pybind11", - "pybind11.share.pkgconfig", - ], - package_data={ - "pybind11": ["py.typed"], - "pybind11.include.pybind11": ["*.h"], - "pybind11.include.pybind11.detail": ["*.h"], - "pybind11.include.pybind11.eigen": ["*.h"], - "pybind11.include.pybind11.stl": ["*.h"], - "pybind11.share.cmake.pybind11": ["*.cmake"], - "pybind11.share.pkgconfig": ["*.pc"], - }, - extras_require={ - "global": ["pybind11_global==$version"] - }, - entry_points={ - "console_scripts": [ - "pybind11-config = pybind11.__main__:main", - ], - "pipx.run": [ - "pybind11 = pybind11.__main__:main", - ] - }, - cmdclass=cmdclass -) diff --git a/lib/pybind11/tools/test-pybind11GuessPythonExtSuffix.cmake b/lib/pybind11/tools/test-pybind11GuessPythonExtSuffix.cmake new file mode 100644 index 0000000..1976abf --- /dev/null +++ b/lib/pybind11/tools/test-pybind11GuessPythonExtSuffix.cmake @@ -0,0 +1,185 @@ +cmake_minimum_required(VERSION 3.15...4.2) + +# Tests for pybind11_guess_python_module_extension +# Run using `cmake -P tools/test-pybind11GuessPythonExtSuffix.cmake` + +include("${CMAKE_CURRENT_LIST_DIR}/pybind11GuessPythonExtSuffix.cmake") + +macro(expect_streq actual expected) + if(NOT "${actual}" STREQUAL "${expected}") + message(SEND_ERROR "Fail\n *** actual: '${actual}'\n *** expected: '${expected}'") + endif() +endmacro() + +macro(expect_false actual) + if("${actual}") + message(SEND_ERROR "Fail\n *** actual: '${actual}'\n *** expected: false") + endif() +endmacro() + +macro(expect_true actual) + if(NOT "${actual}") + message(SEND_ERROR "Fail\n *** actual: '${actual}'\n *** expected: true") + endif() +endmacro() + +# Windows +set(CMAKE_SYSTEM_NAME "Windows") +set(CMAKE_SHARED_MODULE_SUFFIX ".dll") + +set(Python3_SOABI "") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".pyd") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_false("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +set(Python3_SOABI "cp311-win_arm64") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".cp311-win_arm64.pyd") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_false("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +set(Python3_SOABI "cp311d-win_arm64") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".cp311d-win_arm64.pyd") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_true("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +set(Python3_SOABI "pypy310-pp73-win_amd64") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".pypy310-pp73-win_amd64.pyd") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_false("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +set(Python3_SOABI "_d.cp311-win_amd64.pyd") # This is a quirk of FindPython3 +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".cp311-win_amd64.pyd") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "_d") +expect_true("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +unset(Python3_SOABI) +set(ENV{SETUPTOOLS_EXT_SUFFIX} ".cp39-win_arm64.pyd") # Set by cibuildwheel +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".cp39-win_arm64.pyd") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_false("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) +unset(ENV{SETUPTOOLS_EXT_SUFFIX}) + +set(Python3_SOABI "cp311-win_arm64") +set(ENV{SETUPTOOLS_EXT_SUFFIX} "") # Should not be used +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".cp311-win_arm64.pyd") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_false("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) +unset(ENV{SETUPTOOLS_EXT_SUFFIX}) + +# Check the priority of the possible suffix sources. +set(ENV{SETUPTOOLS_EXT_SUFFIX} ".from-setuptools.pyd") +set(SKBUILD_SOABI "from-skbuild") +set(Python3_SOABI "from-python3") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".from-setuptools.pyd") + +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) +unset(ENV{SETUPTOOLS_EXT_SUFFIX}) +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".from-skbuild.pyd") + +unset(SKBUILD_SOABI) +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".from-python3.pyd") + +set(Python3_SOABI "") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".pyd") + +unset(Python3_SOABI) +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" "") + +# macOS +set(CMAKE_SYSTEM_NAME "Darwin") +set(CMAKE_SHARED_MODULE_SUFFIX ".so") + +set(Python3_SOABI "") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".so") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_false("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +set(Python3_SOABI "cpython-312-darwin") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".cpython-312-darwin.so") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_false("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +set(Python3_SOABI "cpython-312d-darwin") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".cpython-312d-darwin.so") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_true("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +# Linux +set(CMAKE_SYSTEM_NAME "Linux") +set(CMAKE_SHARED_MODULE_SUFFIX ".so") + +set(Python3_SOABI "") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".so") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_false("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +set(Python3_SOABI "cpython-312-arm-linux-gnueabihf") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".cpython-312-arm-linux-gnueabihf.so") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_false("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +set(Python3_SOABI "cpython-312d-arm-linux-gnueabihf") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".cpython-312d-arm-linux-gnueabihf.so") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_true("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +set(Python3_SOABI "pypy310-pp73-x86_64-linux-gnu") +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".pypy310-pp73-x86_64-linux-gnu.so") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_false("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) + +set(Python3_SOABI "pypy310d-pp73-x86_64-linux-gnu") +# TODO: I'm not sure if this is the right SOABI for PyPy debug builds +pybind11_guess_python_module_extension("Python3") +expect_streq("${PYTHON_MODULE_EXTENSION}" ".pypy310d-pp73-x86_64-linux-gnu.so") +expect_streq("${PYTHON_MODULE_DEBUG_POSTFIX}" "") +expect_true("${PYTHON_IS_DEBUG}") +unset(PYTHON_MODULE_EXT_SUFFIX) +unset(PYTHON_MODULE_EXT_SUFFIX CACHE) diff --git a/setup.py b/setup.py index b8a96d9..68e951b 100644 --- a/setup.py +++ b/setup.py @@ -137,7 +137,10 @@ def build_extensions(self) -> None: cmake_args = [ f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}{os.sep}", + # PYTHON_EXECUTABLE is the legacy FindPythonLibsNew name; pybind11 3.x + # uses FindPython, which reads Python_EXECUTABLE. Pass both. f"-DPYTHON_EXECUTABLE={sys.executable}", + f"-DPython_EXECUTABLE={sys.executable}", f"-DCMAKE_BUILD_TYPE={cfg}", # not used on MSVC, but no harm ] From cb623095acf28c76071b0f184413e85a6bf274a4 Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Fri, 4 Sep 2026 16:31:01 +0200 Subject: [PATCH 17/30] build: ship the C++ tree in the sdist The sdist has never been buildable. CMakeExtension declares sources=[], so setuptools has no way to infer that lib/ace, lib/pybind11, CMakeLists.txt and src/pyace/**/*.cpp are part of the build, and MANIFEST.in never listed them -- the only MANIFEST.in this repo ever had was the two-line versioneer one from the initial release commit, removed in 0366094 along with the two files it referenced. Measured before this change: pyace-0.4.0.tar.gz 88 entries, 0 .cpp, 0 .h, no lib/, no CMakeLists.txt That matters for two reasons. `python -m build --sdist` followed by building wheels from the sdist is the standard cibuildwheel flow, and it would fail at cmake for every target. And the sdist is the fallback for any platform or python we publish no wheel for: that path should be slow, not broken. After: python-ace-0.4.0.tar.gz 402 entries, 68 .cpp, 148 .h, 10 CMakeLists.txt lib/ace and lib/pybind11 are grafted wholesale rather than enumerated by extension: those trees carry CMake fragments, LICENSE files, .clang-format and yaml-cpp's own build files, and pattern lists are how a file goes missing again later. _maxvol.pyx is listed explicitly because maxvol_extension() only registers it as an Extension source when Cython is importable, which would otherwise make the sdist's contents depend on the environment that built it. A graft copies what is on disk rather than what git tracks, so a release sdist must be built from a clean checkout. Speculative global-exclude guards against in-tree build output were dropped: they matched nothing on a clean tree and emitted nine "no previously-included files matching" warnings, which would have undone 89b9ca4. --- MANIFEST.in | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..2a3c978 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,25 @@ +# What the sdist needs in order to build. +# +# None of this is inferable by setuptools: CMakeExtension declares sources=[], +# so nothing under lib/ or src/pyace/*.cpp is picked up automatically, and an +# sdist without them cannot build at all. That is the fallback path for every +# platform and python we publish no wheel for, so it has to work. + +include CMakeLists.txt + +# Vendored dependencies, all compiled from source as part of the extension +# build: ace + ace-evaluator (which carry yaml-cpp, cnpy and wigner-cpp) and +# pybind11. Grafted wholesale rather than by extension -- these trees contain +# CMake fragments, .clang-format, LICENSE files and yaml-cpp's own build files, +# and enumerating them by pattern is how a file goes missing again later. +# +# A graft copies what is on disk, not what git tracks, so build a release sdist +# from a clean checkout -- an in-tree cmake build would otherwise be swept in. +graft lib/ace +graft lib/pybind11 + +# Binding translation units, plus the cythonised maxvol kernel. _maxvol.pyx +# needs listing explicitly: maxvol_extension() in setup.py only registers it as +# an Extension source when Cython is importable, so relying on that would make +# the sdist's contents depend on the environment it was built in. +recursive-include src/pyace *.cpp *.h *.pyx From 7ada8e440b2feb8ab1821e1a1cb5bd87bb281b29 Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Fri, 4 Sep 2026 16:31:12 +0200 Subject: [PATCH 18/30] build: distribute as python-ace, keep importing as pyace python-ace is the only name available to us on PyPI, so that is what the distribution has to be called; the import package stays pyace. The split is ordinary -- scikit-learn imports as sklearn, pillow as PIL -- and affects nothing but the [project] name and one lookup. That lookup is the subtle part. importlib.metadata.version() resolves a *distribution* name, not an import package, so __init__ has to ask for "python-ace" even though the surrounding module is pyace. Left as "pyace" it raises PackageNotFoundError and silently reports __version__ == "unknown" for every installed user, which is why the comment is there. Verified from the built sdist: PKG-INFO Name: python-ace Version: 0.4.0 top_level.txt pyace [project.scripts], packages.find and the pyace.data / pyace.maxvol package-data keys are all import paths and are deliberately untouched. Note python-ace 0.2.8 already exists on PyPI as a single hand-built cp39 wheel, and an unrelated pyace 0.1.1 sdist also exists; neither is produced by this tree. --- pyproject.toml | 2 +- src/pyace/__init__.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a98170c..603044c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ requires = [ build-backend = "setuptools.build_meta" [project] -name = "pyace" +name = "python-ace" version = "0.4.0" authors = [ { name = "Yury Lysogorskiy", email = "yury.lysogorskiy@rub.de" }, diff --git a/src/pyace/__init__.py b/src/pyace/__init__.py index 038b2eb..fc06980 100644 --- a/src/pyace/__init__.py +++ b/src/pyace/__init__.py @@ -41,6 +41,7 @@ from importlib.metadata import PackageNotFoundError, version as _package_version try: - __version__ = _package_version("pyace") + # The distribution on PyPI is "python-ace"; the import package is "pyace". + __version__ = _package_version("python-ace") except PackageNotFoundError: # imported from a source tree that was never installed __version__ = "unknown" From c5f6df3edb3951092a96dff1dcd0ef3ec0fd7c81 Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Fri, 4 Sep 2026 16:31:25 +0200 Subject: [PATCH 19/30] build: add PYACE_PORTABLE_MATH so wheels can reach manylinux_2_17 -Ofast implies -ffast-math, which lets gcc route exp/log/pow through glibc's vector math library. Measured on a built evaluator module: 92 _ZGV* symbols plus __exp_finite/__pow_finite, a libmvec.so.1 dependency, and a floor of GLIBC_2.22. auditwheel's manylinux_2_17 policy caps GLIBC at 2.17 and does not whitelist libmvec (2_28 does), so with -Ofast the lowest tag a wheel can carry is manylinux_2_28. Confirmed with a minimal translation unit: g++ -Ofast libmvec: yes 3 x _ZGV* g++ -O3 -fno-fast-math libmvec: no 0 x _ZGV* The option is OFF by default, so building from source -- locally, on HPC -- keeps -Ofast and is unchanged. Only redistributable builds turn it on and trade vectorised transcendental math for a glibc 2.17 floor, which adds RHEL/CentOS 7 to the reachable set. The cost of that trade has not been measured yet. The radial functions are exactly the exp/pow-heavy code -ffast-math was helping, so wheels may be measurably slower than a source build; compare with pace_timing before assuming the trade is worth it. If it is not, reverting means dropping PYACE_PORTABLE_MATH from the wheel job and restoring the manylinux_2_28 image. --- CMakeLists.txt | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 25396c3..072d28e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -171,7 +171,26 @@ set(GRACE_FS_SOURCES # C++ FLAGS #--------------------------------------------------------- #set(CMAKE_CXX_FLAGS "-Wall -Wextra") -set(CMAKE_CXX_FLAGS_RELEASE "-Ofast -DNDEBUG") + +# -Ofast implies -ffast-math, which lets gcc route exp/log/pow through glibc's +# vector math library (libmvec: _ZGVbN2v_exp and friends) and the __*_finite +# aliases. That is fast, but it raises the floor of the resulting binary to +# GLIBC_2.22 and adds a libmvec.so.1 dependency -- and auditwheel's +# manylinux_2_17 policy caps GLIBC at 2.17 and does not whitelist libmvec, so a +# wheel built this way cannot be tagged below manylinux_2_28. +# +# Redistributable builds turn this option on to trade vectorised transcendental +# math for that lower floor. Builds for a known machine -- local, HPC -- should +# leave it off and keep -Ofast. This is why wheels may be measurably slower than +# a source build; measure with `pace_timing` before assuming either way. +option(PYACE_PORTABLE_MATH + "Drop -ffast-math to keep the glibc floor low enough for manylinux_2_17" OFF) + +if (PYACE_PORTABLE_MATH) + set(CMAKE_CXX_FLAGS_RELEASE "-O3 -fno-fast-math -DNDEBUG") +else () + set(CMAKE_CXX_FLAGS_RELEASE "-Ofast -DNDEBUG") +endif () # keep Debug builds debuggable (only used with -DCMAKE_BUILD_TYPE=Debug / DEBUG=1) set(CMAKE_CXX_FLAGS_DEBUG "-Og -g") From a45bfb4f8e99f039d5708e03a74774c700407a7a Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Fri, 4 Sep 2026 16:31:44 +0200 Subject: [PATCH 20/30] CI: build manylinux wheels; make every publish manual Adds wheels.yml: cibuildwheel 4.2.0, linux x86_64, cp39-cp313, built in the manylinux2014 image with PYACE_PORTABLE_MATH=ON so auditwheel can tag the result manylinux_2_17. Publishing is never automatic. A PyPI filename can be uploaded exactly once -- deleting or yanking does not free the version -- so a botched 0.4.0 would burn 0.4.0 permanently. Pushes to master or feature/** and pull requests build cp312 only and upload artifacts; the full matrix and any upload require a manual workflow_dispatch with an explicit publish target, defaulting to none. There is deliberately no tag or release trigger. Uploads use trusted publishing, so no API token is stored. Note workflow_dispatch only appears in the Actions UI once this file is on the default branch, which is a GitHub restriction. The push trigger is what makes the build reachable from a feature branch before then; the publish job stays gated on workflow_dispatch, so a push can never upload anything. The image choice is constrained from both sides. Below: -Ofast's libmvec dependency, hence the portable-math flag. Above: _maxvol.pyx cimports scipy.linalg.cython_blas, so numpy and scipy must install *inside* the container. Current releases are manylinux_2_28-only, but pip skips those as incompatible and resolves back to the newest 2_17 build -- numpy 2.0.2-2.2.x and scipy 1.13-1.16 all publish 2_17 wheels for cp39-cp313, as do cmake and ninja, and Cython and setuptools are pure python. Runtime is unaffected. manylinux_2_17 is also the floor: no manylinux_2_14 policy exists, and 2_12 (CentOS 6) has retired images and no modern numpy/scipy wheels. CIBW_TEST_COMMAND checks four things against the installed wheel, each of which has actually been broken here: 1. __version__ != "unknown", i.e. the python-ace distribution lookup works 2. all seven pybind11 modules import, covering the 2.11 -> 3.1 upgrade 3. pyace.maxvol.c_maxvol exists and runs, so the compiled kernel shipped rather than silently falling back to py_maxvol 4. a console entry point resolves, after bin/ became [project.scripts] Both jobs also assert the sdist carries CMakeLists.txt, pybind11 and >=60 .cpp files, so cb62309 cannot silently regress. publish-to-test-pypi.yml: was on: push against a retired ubuntu-18.04 runner, so it was inert -- but had the runner ever been updated it would have published on every commit. Now workflow_dispatch only, with modernised actions. It no longer builds a wheel: `build --wheel` on a bare runner emits a linux_x86_64 tag, which PyPI and TestPyPI both reject, and that is not fixable with a flag -- portable wheels need the container. skip-existing avoids a duplicate-filename failure now that wheels.yml can publish the same sdist. --- .github/workflows/publish-to-test-pypi.yml | 78 ++++++----- .github/workflows/wheels.yml | 148 +++++++++++++++++++++ 2 files changed, 193 insertions(+), 33 deletions(-) create mode 100644 .github/workflows/wheels.yml diff --git a/.github/workflows/publish-to-test-pypi.yml b/.github/workflows/publish-to-test-pypi.yml index c477857..f751be9 100644 --- a/.github/workflows/publish-to-test-pypi.yml +++ b/.github/workflows/publish-to-test-pypi.yml @@ -1,36 +1,48 @@ -name: Publish to TestPyPI -on: push +name: Publish sdist to TestPyPI + +# Manual only. An index filename can be uploaded exactly once -- deleting or +# yanking does not free it -- so this must never fire by itself. It used to run +# `on: push`, which would have burned a version number on every commit. +on: workflow_dispatch + jobs: build-n-publish: - name: Build and publish Python distributions to TestPyPI - runs-on: ubuntu-18.04 + name: Build and publish the source distribution to TestPyPI + runs-on: ubuntu-latest steps: - - uses: actions/checkout@master - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Install pypa/build - run: >- - python -m - pip install - build - --user - - name: Build a binary wheel and a source tarball - run: >- - python -m - build - --sdist - --wheel - --outdir dist/ - . - - name: Publish distribution to Test PyPI - uses: pypa/gh-action-pypi-publish@master - with: - password: ${{ secrets.TEST_PYPI_API_TOKEN }} - repository_url: https://test.pypi.org/legacy/ - # - name: Publish distribution 📦 to PyPI - # if: startsWith(github.ref, 'refs/tags') - # uses: pypa/gh-action-pypi-publish@master - # with: - # password: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + # sdist only, deliberately. `build --wheel` on a bare runner emits a + # linux_x86_64 tag, which PyPI and TestPyPI both reject -- portable wheels + # have to be built inside a manylinux container, which is what + # wheels.yml does with cibuildwheel. Publishing wheels from here is not a + # matter of adding a flag; it cannot work. + - name: Build the source distribution + run: pipx run build --sdist + + # Same guard as wheels.yml: CMakeExtension declares sources=[], so the + # C++ tree only reaches the sdist via MANIFEST.in. An sdist without it + # uploads happily and then fails for whoever builds from source. + - name: Verify the sdist can actually build + run: | + set -euo pipefail + tar tzf dist/*.tar.gz > files.txt + grep -qE 'CMakeLists\.txt$' files.txt || { echo "sdist has no CMakeLists.txt"; exit 1; } + grep -qE 'lib/pybind11/include/pybind11/pybind11\.h$' files.txt || { echo "sdist has no pybind11"; exit 1; } + cpp=$(grep -cE '\.cpp$' files.txt) + echo "sdist carries $cpp .cpp files" + [ "$cpp" -ge 60 ] || { echo "expected >=60 .cpp files, got $cpp"; exit 1; } + + - name: Publish to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + repository-url: https://test.pypi.org/legacy/ + # wheels.yml can also publish this same sdist to TestPyPI. Without + # this, whichever runs second fails on the duplicate filename. + skip-existing: true diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 0000000..e4c782e --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,148 @@ +name: Build wheels + +# Publishing is deliberately never automatic. A PyPI filename can only be +# uploaded once -- deleting or yanking a release does NOT free the version +# number -- so a botched 0.4.0 would burn 0.4.0 permanently. Every trigger here +# only produces downloadable artifacts; uploading is a separate manual run with +# an explicit target, and the recommended first target is TestPyPI. +on: + # Branch pushes and pull requests both give a build without touching master. + # Note workflow_dispatch below only appears in the Actions UI once this file + # is on the default branch -- that is a GitHub restriction, not a choice + # here -- so until then, pushing to one of these branches is how you run it. + push: + branches: + - master + - "feature/**" + pull_request: + workflow_dispatch: + inputs: + publish: + description: "Upload the built distributions where?" + type: choice + default: none + options: [none, testpypi, pypi] + +jobs: + sdist: + name: sdist + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - run: pipx run build --sdist + + # The sdist shipped no C++ at all from the initial release commit until + # MANIFEST.in was added: CMakeExtension declares sources=[], so setuptools + # cannot infer the build tree. It is invisible in a wheel-only world and + # only bites whoever falls back to building from source, so assert it here + # rather than find out from a bug report. + - name: Verify the sdist can actually build + run: | + set -euo pipefail + tar tzf dist/*.tar.gz > files.txt + for pattern in \ + 'CMakeLists\.txt$' \ + 'lib/pybind11/include/pybind11/pybind11\.h$' \ + 'lib/ace/ace-evaluator/CMakeLists\.txt$' \ + 'src/pyace/maxvol/_maxvol\.pyx$' + do + grep -qE "$pattern" files.txt || { echo "sdist is missing: $pattern"; exit 1; } + done + cpp=$(grep -cE '\.cpp$' files.txt) + echo "sdist carries $cpp .cpp files" + [ "$cpp" -ge 60 ] || { echo "expected >=60 .cpp files, got $cpp"; exit 1; } + + - uses: actions/upload-artifact@v4 + with: + name: sdist + path: dist/*.tar.gz + + wheels: + name: wheels (linux x86_64) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build wheels + uses: pypa/cibuildwheel@v4.2.0 + env: + CIBW_ARCHS_LINUX: x86_64 + # Only a deliberate manual run builds all five interpreters; pushes and + # pull requests build one, so routine commits cost a single build + # rather than five. Upper bound is pybind11 3.1, which supports 3.9 + # through 3.14; the floor matches requires-python. + CIBW_BUILD: >- + ${{ github.event_name == 'workflow_dispatch' + && 'cp39-manylinux_x86_64 cp310-manylinux_x86_64 cp311-manylinux_x86_64 cp312-manylinux_x86_64 cp313-manylinux_x86_64' + || 'cp312-manylinux_x86_64' }} + + # glibc 2.17 -- the lowest floor this project can reach, which needs + # two things to line up: + # + # * PYACE_PORTABLE_MATH below. Default -Ofast pulls in libmvec and + # GLIBC_2.22, and the manylinux_2_17 policy caps GLIBC at 2.17 and + # does not whitelist libmvec, so auditwheel would refuse the tag. + # * build-time numpy/scipy (needed for the cython_blas cimport in + # _maxvol.pyx) must be installable *inside* this image. The newest + # releases are manylinux_2_28-only, but pip skips those as + # incompatible and resolves back to the newest 2_17 build -- + # numpy 2.0.2-2.2.x and scipy 1.13-1.16 all publish 2_17 wheels for + # cp39-cp313. Runtime is unaffected: users still get current + # numpy/scipy, which are forward-compatible with these headers. + # + # auditwheel enforces all of this: if a module ends up needing a + # symbol above the policy, the build fails rather than shipping a + # wheel that dies with a link error on an old machine. + CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 + + # Only wheels are built portable; a source build keeps -Ofast. + CIBW_ENVIRONMENT: CMAKE_ARGS="-DPYACE_PORTABLE_MATH=ON" + + # Each of these has actually been broken in this project at some point, + # so the wheel is not accepted until all four hold. + # 1. importlib.metadata looks up the DISTRIBUTION name (python-ace), + # not the import package (pyace) -- "unknown" means that broke. + # 2. all seven pybind11 modules load (pybind11 2.11 -> 3.1 upgrade). + # 3. the compiled maxvol kernel is present, not the slow scipy + # fallback -- it silently vanished from wheels before. + # 4. a console entry point resolves, since bin/ scripts became + # [project.scripts]. + CIBW_TEST_COMMAND: >- + python -c "import pyace; print('version:', pyace.__version__); assert pyace.__version__ != 'unknown', 'distribution-name lookup failed'" + && python -c "import pyace.sharmonics, pyace.coupling, pyace.basis, pyace.evaluator, pyace.catomicenvironment, pyace.calculator, pyace.grace_fs; print('7 pybind11 modules ok')" + && python -c "from pyace.maxvol import c_maxvol; import numpy as np; i, c = c_maxvol(np.random.rand(64, 8), 1.05, 100, -1, 0); print('compiled maxvol ok, selected', len(i), 'rows')" + && pace_info --help > /dev/null && echo 'entry points ok' + + - uses: actions/upload-artifact@v4 + with: + name: wheels-linux-x86_64 + path: wheelhouse/*.whl + + publish: + name: publish to ${{ github.event.inputs.publish }} + needs: [sdist, wheels] + if: github.event_name == 'workflow_dispatch' && github.event.inputs.publish != 'none' + runs-on: ubuntu-latest + environment: ${{ github.event.inputs.publish }} + permissions: + id-token: write # trusted publishing; no API token in secrets + steps: + - uses: actions/download-artifact@v4 + with: + path: dist + merge-multiple: true + + - run: ls -l dist/ + + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: >- + ${{ github.event.inputs.publish == 'testpypi' + && 'https://test.pypi.org/legacy/' + || 'https://upload.pypi.org/legacy/' }} From 0c228fb9c41af7cdf77f18311395a8dbb698dbea Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Fri, 4 Sep 2026 16:40:36 +0200 Subject: [PATCH 21/30] CI: build wheels on manylinux_2_28, not manylinux2014 The manylinux2014 build failed: pip downloaded numpy-2.5.2.tar.gz and died on "NumPy requires GCC >= 10.3" inside the image. a45bfb4 justified the 2_17 image by claiming pip would treat the newer manylinux_2_28-only releases as incompatible and resolve back to the newest 2_17 build. That is wrong. pip prefers the newest *version* and falls back to that version's sdist; it backtracks on dependency conflicts, not on wheel availability. So numpy>=2.0,<3 selected 2.5.2 and tried to compile it. Pinning build-time numpy/scipy to versions that do ship 2_17 wheels would not rescue it, because cibuildwheel installs the runtime dependencies in the same container to run CIBW_TEST_COMMAND, and pandas 3.0.5 and scikit-learn 1.9.0 publish x86_64 wheels only for manylinux_2_28. The same applies outside CI: a glibc-2.17 user running `pip install python-ace` would get our wheel and then a source build of pandas. There is no reach to be had below 2_28 while numpy, scipy, pandas and scikit-learn all stop there -- 2_28 is the ecosystem floor, covering RHEL/Alma/Rocky 8+, Ubuntu 20.04+ and Debian 10+. PYACE_PORTABLE_MATH is consequently no longer passed to the wheel build: manylinux_2_28 whitelists libmvec and permits GLIBC_2.22, so -Ofast is free here and the wheels keep the vectorised transcendental math. The option remains in CMakeLists.txt, unused by CI, documenting the -Ofast/libmvec/glibc relationship for whenever a lower floor is worth chasing. --- .github/workflows/wheels.yml | 37 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index e4c782e..ac90505 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -81,27 +81,26 @@ jobs: && 'cp39-manylinux_x86_64 cp310-manylinux_x86_64 cp311-manylinux_x86_64 cp312-manylinux_x86_64 cp313-manylinux_x86_64' || 'cp312-manylinux_x86_64' }} - # glibc 2.17 -- the lowest floor this project can reach, which needs - # two things to line up: + # glibc 2.28 -- RHEL/Alma/Rocky 8+, Ubuntu 20.04+, Debian 10+. This is + # the floor the dependency set allows, not a compromise. # - # * PYACE_PORTABLE_MATH below. Default -Ofast pulls in libmvec and - # GLIBC_2.22, and the manylinux_2_17 policy caps GLIBC at 2.17 and - # does not whitelist libmvec, so auditwheel would refuse the tag. - # * build-time numpy/scipy (needed for the cython_blas cimport in - # _maxvol.pyx) must be installable *inside* this image. The newest - # releases are manylinux_2_28-only, but pip skips those as - # incompatible and resolves back to the newest 2_17 build -- - # numpy 2.0.2-2.2.x and scipy 1.13-1.16 all publish 2_17 wheels for - # cp39-cp313. Runtime is unaffected: users still get current - # numpy/scipy, which are forward-compatible with these headers. - # - # auditwheel enforces all of this: if a module ends up needing a - # symbol above the policy, the build fails rather than shipping a - # wheel that dies with a link error on an old machine. - CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 + # An earlier revision tried manylinux2014 (glibc 2.17) on the theory + # that pip would resolve back to an older numpy/scipy that publishes + # 2_17 wheels. It does not: pip prefers the newest *version* and falls + # back to its sdist, so it downloaded numpy 2.5.2.tar.gz and died on + # "NumPy requires GCC >= 10.3". Pinning around that would not help + # either, because cibuildwheel installs the runtime dependencies in + # the same container to run the tests, and current pandas and + # scikit-learn publish x86_64 wheels only for 2_28 -- as would any + # actual glibc-2.17 user, who would get our wheel followed by a source + # build of pandas. There is no reach to be had below 2_28 while + # numpy, scipy, pandas and scikit-learn all stop there. + CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 - # Only wheels are built portable; a source build keeps -Ofast. - CIBW_ENVIRONMENT: CMAKE_ARGS="-DPYACE_PORTABLE_MATH=ON" + # PYACE_PORTABLE_MATH is deliberately NOT set: 2_28 whitelists + # libmvec and permits GLIBC_2.22, so -Ofast costs nothing here. The + # option stays in CMakeLists.txt for whenever a lower floor is worth + # chasing again. # Each of these has actually been broken in this project at some point, # so the wheel is not accepted until all four hold. From 5beead75920ed395a7731ce687d6662b491b67df Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Fri, 4 Sep 2026 16:43:54 +0200 Subject: [PATCH 22/30] CI: bump actions to their current majors The first wheels.yml run warned that actions/checkout@v4 targets Node 20, which is deprecated and being forced onto Node 24. The pins were further behind than that warning suggested: actions/checkout v4 -> v7 actions/setup-python v5 -> v7 actions/upload-artifact v4 -> v7 actions/download-artifact v4 -> v8 All four now run on node24, which clears the deprecation. Every input this repo passes -- name, path, merge-multiple, python-version -- still exists in the new majors, checked against each action.yml before bumping; upload v7 and download v8 are both post-v4 immutable-artifact versions, so they interoperate. pypa/cibuildwheel@v4.2.0 and pypa/gh-action-pypi-publish@release/v1 are already current. test.yml still pins checkout@v3/@v2 and setup-python@v3 and is left alone here -- it is a different workflow with its own tensorpotential setup, worth bumping separately. --- .github/workflows/publish-to-test-pypi.yml | 4 ++-- .github/workflows/wheels.yml | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/publish-to-test-pypi.yml b/.github/workflows/publish-to-test-pypi.yml index f751be9..766f35c 100644 --- a/.github/workflows/publish-to-test-pypi.yml +++ b/.github/workflows/publish-to-test-pypi.yml @@ -10,10 +10,10 @@ jobs: name: Build and publish the source distribution to TestPyPI runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: python-version: "3.12" diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index ac90505..4799f0e 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -28,9 +28,9 @@ jobs: name: sdist runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: python-version: "3.12" @@ -57,7 +57,7 @@ jobs: echo "sdist carries $cpp .cpp files" [ "$cpp" -ge 60 ] || { echo "expected >=60 .cpp files, got $cpp"; exit 1; } - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: sdist path: dist/*.tar.gz @@ -66,7 +66,7 @@ jobs: name: wheels (linux x86_64) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Build wheels uses: pypa/cibuildwheel@v4.2.0 @@ -117,7 +117,7 @@ jobs: && python -c "from pyace.maxvol import c_maxvol; import numpy as np; i, c = c_maxvol(np.random.rand(64, 8), 1.05, 100, -1, 0); print('compiled maxvol ok, selected', len(i), 'rows')" && pace_info --help > /dev/null && echo 'entry points ok' - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: wheels-linux-x86_64 path: wheelhouse/*.whl @@ -131,7 +131,7 @@ jobs: permissions: id-token: write # trusted publishing; no API token in secrets steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: path: dist merge-multiple: true From b71416cd4b8e69c93e4774d4f998dcf755310a28 Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Fri, 4 Sep 2026 16:51:14 +0200 Subject: [PATCH 23/30] CI: allow the full python matrix from a branch push [full-matrix] Everything verified so far is cp312 only. cp39 and cp313 are the two worth actual doubt: cp39 resolves numpy to exactly 2.0.2, the only release in >=2.0,<3 that publishes cp39 wheels, and cp313 is the reason pybind11 was upgraded to 3.1 in the first place. A matrix entry that fails after a successful upload cannot be fixed at the same version, so all five want exercising before anything is published. The full matrix was reachable only via workflow_dispatch, which GitHub does not surface until the workflow file is on the default branch. Putting [full-matrix] in a pushed commit's message now selects it too, so all five can be built from a feature branch without merging anything to master first. Routine commits still build one interpreter. head_commit is absent on pull_request events, where contains() evaluates false, so pull requests keep the single-interpreter build. This commit's own message carries the marker, so the push that delivers the change also runs the matrix it enables. --- .github/workflows/wheels.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 4799f0e..7b77415 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -72,12 +72,18 @@ jobs: uses: pypa/cibuildwheel@v4.2.0 env: CIBW_ARCHS_LINUX: x86_64 - # Only a deliberate manual run builds all five interpreters; pushes and - # pull requests build one, so routine commits cost a single build - # rather than five. Upper bound is pybind11 3.1, which supports 3.9 - # through 3.14; the floor matches requires-python. + # Routine commits build one interpreter; all five are built only when + # asked for, so a normal push costs one build rather than five. Ask + # either with a manual run, or -- since workflow_dispatch is invisible + # until this file reaches the default branch -- by putting + # [full-matrix] in the pushed commit's message. + # + # Upper bound is pybind11 3.1, which supports 3.9 through 3.14; the + # floor matches requires-python. head_commit is absent on + # pull_request events, where contains() is simply false. CIBW_BUILD: >- - ${{ github.event_name == 'workflow_dispatch' + ${{ (github.event_name == 'workflow_dispatch' + || contains(github.event.head_commit.message, '[full-matrix]')) && 'cp39-manylinux_x86_64 cp310-manylinux_x86_64 cp311-manylinux_x86_64 cp312-manylinux_x86_64 cp313-manylinux_x86_64' || 'cp312-manylinux_x86_64' }} From 60a0e357feda0a8f99deb8b5dc24ee0698717c2d Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Fri, 4 Sep 2026 17:07:35 +0200 Subject: [PATCH 24/30] build: keep C++ sources out of the installed wheel cb62309 taught MANIFEST.in about the C++ tree so the sdist could build. Those sources live inside the package tree, and include-package-data defaults to true when metadata comes from pyproject.toml, so they were also installed into every wheel. Inspecting cp312 from the first full-matrix run: 17 x .cpp/.h (ace/, ace-evaluator/, utils/) pyace/maxvol/_maxvol.pyx and the generated _maxvol.c pyace/ace-evaluator/, which has no __init__.py and is not importable Harmless but wrong: dead weight in every install, and a package directory that exists only because a graft pattern matched it. Setting include-package-data = false splits the two concerns properly -- MANIFEST.in governs the sdist, [tool.setuptools.package-data] governs what ships installed. The three files that genuinely belong at runtime (pyace/data/input_template.yaml, the rawlsLS pickle, and the vendored maxvol MIT notice) are all already listed there, so nothing is lost. Verified the sdist is unchanged: still 68 .cpp, 148 .h, 1 .pyx, 10 CMakeLists.txt. PEP 639 license files are unaffected -- those come from license-files and land in .dist-info/licenses/, not from package data. Adds a wheel guard alongside the existing sdist one: every wheel must contain zero .cpp/.h/.pyx/.c and exactly 8 .so (seven pybind11 modules plus _maxvol). The .so count also catches a module silently failing to build, which is how the maxvol extension used to disappear. --- .github/workflows/wheels.yml | 16 ++++++++++++++++ pyproject.toml | 8 ++++++++ 2 files changed, 24 insertions(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 7b77415..850ac33 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -123,6 +123,22 @@ jobs: && python -c "from pyace.maxvol import c_maxvol; import numpy as np; i, c = c_maxvol(np.random.rand(64, 8), 1.05, 100, -1, 0); print('compiled maxvol ok, selected', len(i), 'rows')" && pace_info --help > /dev/null && echo 'entry points ok' + # MANIFEST.in names the C++ sources so the sdist can build, and they sit + # inside the package tree -- so a lapse in include-package-data silently + # installs all of them into every wheel. It happened once already. + - name: Verify the wheels ship no source + run: | + set -euo pipefail + for whl in wheelhouse/*.whl; do + bad=$(unzip -Z1 "$whl" | grep -E '\.(cpp|h|pyx|c)$' || true) + if [ -n "$bad" ]; then + echo "$(basename "$whl") ships source files:"; echo "$bad"; exit 1 + fi + n=$(unzip -Z1 "$whl" | grep -cE '\.so$') + echo "$(basename "$whl"): $n compiled modules, no source" + [ "$n" -eq 8 ] || { echo "expected 8 .so (7 pybind11 + _maxvol), got $n"; exit 1; } + done + - uses: actions/upload-artifact@v7 with: name: wheels-linux-x86_64 diff --git a/pyproject.toml b/pyproject.toml index 603044c..f150942 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,14 @@ pace_yaml2yace = "pyace.cli.pace_yaml2yace:main" [tool.setuptools] zip-safe = false +# MANIFEST.in has to name the C++ sources so the sdist can build, but those +# live inside the package tree, and include-package-data (which defaults to +# true here) would then install them into every wheel as well -- 17 .cpp/.h +# files, the generated _maxvol.c, and pyace/ace-evaluator/, which is not even +# an importable package. Turning it off makes MANIFEST.in govern the sdist and +# the package-data table below govern what ships installed; sdist contents are +# unaffected. Anything a user needs at runtime must be listed there. +include-package-data = false [tool.setuptools.packages.find] where = ["src"] From 3fb0993fb677d89e00370ef8b4855bd378345968 Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Fri, 4 Sep 2026 17:26:33 +0200 Subject: [PATCH 25/30] ci: rebuild all wheels after package-data fix [full-matrix] From 9a614c2150a3a0d9e970774830a55ad993457254 Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Fri, 4 Sep 2026 17:27:55 +0200 Subject: [PATCH 26/30] ci: rebuild all wheels after the package-data fix [full-matrix] From a30785e1e327064477164cd39e010fb72159c5b3 Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Fri, 4 Sep 2026 18:26:25 +0200 Subject: [PATCH 27/30] build: 0.4.0rc1 for a pre-release to PyPI [full-matrix] A tutorial in the next few weeks needs a `pip install` that works reliably, and TestPyPI cannot provide one: packaging.python.org states its database "may be periodically pruned, so it is not unusual for user accounts to be deleted". Installing from it also requires --extra-index-url, which makes pip take the highest version of *every* dependency across both indexes -- the dependency-confusion pattern, not something to teach. CI artifacts are no help either; they expire after 90 days and need GitHub auth to fetch. A pre-release on the real index solves it without spending 0.4.0. pip does not select pre-releases unless asked, so `pip install python-ace` continues to resolve to 0.2.8 for everyone else, while the tutorial pins `python-ace==0.4.0rc1` and gets a permanent, CDN-backed wheel. Bump back to a final version when the pre-release has been exercised. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f150942..19b7aa5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ build-backend = "setuptools.build_meta" [project] name = "python-ace" -version = "0.4.0" +version = "0.4.0rc1" authors = [ { name = "Yury Lysogorskiy", email = "yury.lysogorskiy@rub.de" }, { name = "Anton Bochkarev" }, From b8cd66a949f98caa81205e13239de19080546726 Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Thu, 29 Oct 2026 23:13:27 +0100 Subject: [PATCH 28/30] Refresh Python and TensorFlow compatibility and documentation --- .github/workflows/test.yml | 32 +++-- .readthedocs.yaml | 1 + docs/pacemaker/faq.md | 4 +- docs/pacemaker/inputfile.md | 2 +- docs/pacemaker/install.md | 208 ++++++++++++++++++------------- docs/pacemaker/quickstart.md | 14 +-- docs/requirements.txt | 6 +- mkdocs.yml | 8 ++ src/pyace/metrics_aggregator.py | 3 +- tests/test_metrics_aggregator.py | 31 +++++ tools/validate_gpu.py | 61 +++++++++ 11 files changed, 253 insertions(+), 117 deletions(-) create mode 100644 tests/test_metrics_aggregator.py create mode 100644 tools/validate_gpu.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 512f43a..41586cf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,7 +5,7 @@ name: Python package on: pull_request: - branches: [ "master" ] + branches: [ "master", "feature/build-speedup" ] jobs: build: @@ -14,36 +14,42 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10"] # ver. 3.11 is not supported by TP + python-version: ["3.10", "3.11", "3.12", "3.13"] + tensorflow-version: ["2.20.0", "2.21.0"] + include: + - python-version: "3.9" + tensorflow-version: "2.20.0" + + env: + CUDA_VISIBLE_DEVICES: "-1" + TF_USE_LEGACY_KERAS: "1" + CMAKE_BUILD_PARALLEL_LEVEL: "2" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install general dependencies run: | python -m pip install --upgrade pip python -m pip install pytest - - name: Checkout tensorpotential - uses: actions/checkout@v2 + - name: Checkout current tensorpotential with PACE compatibility + uses: actions/checkout@v4 with: - repository: ICAMS/TensorPotential + repository: ICAMS/grace-tensorpotential path: tensorpotential - name: Install tensorpotential run: | cd tensorpotential - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - #python setup.py install - pip install . + python -m pip install "tensorflow==${{ matrix.tensorflow-version }}" "tf-keras~=${{ matrix.tensorflow-version }}" . - name: Install python-ace run: | - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - #python setup.py install - pip install . + python -m pip install . - name: Test with python-ace with tensorpotential run: | + python -m pip check python -m pytest tests/ --runtensorpot - name: Integration test of CLI run: | diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 015eb5d..430a487 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -12,6 +12,7 @@ build: mkdocs: configuration: mkdocs.yml + fail_on_warning: true # Optionally declare the Python requirements required to build your docs python: diff --git a/docs/pacemaker/faq.md b/docs/pacemaker/faq.md index b298046..3bc5ac0 100644 --- a/docs/pacemaker/faq.md +++ b/docs/pacemaker/faq.md @@ -6,7 +6,7 @@ If `batch_size_reduction=True` (default option), then automatic batch size reduc ## My fit on GPU crushes with OOM error, what to do ? This means that you are trying to fit at once too much data into the GPU memory. The amount of data processed by GPU -at once is controlled by [batch_size](#backend-specification) parameter, try to reduce it or set `batch_size_reduction=True`. An optimal value for +at once is controlled by [batch_size](inputfile.md#backend_specification) parameter, try to reduce it or set `batch_size_reduction=True`. An optimal value for this parameter is totally empirical as it depends on data, potential configuration and GPU itself. @@ -47,7 +47,7 @@ If you want to continue fit without changing the basis size, you can do the foll doing this will override specifications in the `input.yaml`. -If you want to extend the basis (aka do the [ladder scheme fitting](#inputfile.md#ladder_hiererchical_basis_extension)): +If you want to extend the basis (aka do the [ladder scheme fitting](inputfile.md#ladder_hiererchical_basis_extension)): - Specify your potential as initial potential diff --git a/docs/pacemaker/inputfile.md b/docs/pacemaker/inputfile.md index f7e878c..c6d8db4 100644 --- a/docs/pacemaker/inputfile.md +++ b/docs/pacemaker/inputfile.md @@ -36,7 +36,7 @@ data: ``` Dataset could be saved into file as a pickled `pandas` dataframe with special names -for [columns](#fitting-dataset-preparation). +for [columns](quickstart.md#optional_manual_fitting_dataset_preparation). ```YAML data: diff --git a/docs/pacemaker/install.md b/docs/pacemaker/install.md index 9edc715..108d784 100644 --- a/docs/pacemaker/install.md +++ b/docs/pacemaker/install.md @@ -1,122 +1,150 @@ # Installation -## (Optional) Creating a Separate Environment +`pacemaker` is installed by the `python-ace` package. TensorFlow fitting uses +`tensorpotential.compat.pace` from the current +[GRACE TensorPotential repository](https://github.com/ICAMS/grace-tensorpotential). +The old `ICAMS/TensorPotential` repository requires Python <3.11 and should not +be used for a new installation. -It is a common practice to create a separate Python environment to avoid dependency conflicts. You can create a new environment named `ace` with the minimal required packages using the following command: +## Create an environment +Python 3.12 is a suitable starting point for a new environment: -### Using `mamba` (Recommended) - -[Mamba](https://mamba.readthedocs.io/en/latest/index.html) is a fast, robust, and cross-platform package manager. Check the [Mamba installation guide](https://mamba.readthedocs.io/en/latest/installation/mamba-installation.html) for details. +```bash +mamba create -n ace python=3.12 pip +mamba activate ace +python -m pip install --upgrade pip +``` -Installation steps: -* Download the Miniforge3 distribution from [here](https://github.com/conda-forge/miniforge?tab=readme-ov-file#miniforge3). - -* Run the downloaded file: - ```bash - sh Miniforge3-Linux-x86_64.sh - ``` - and follow the installation instructions. - -* Create the `mamba` environment: - ```bash - mamba create -n ace python=3.9 - ``` +`conda create` and `conda activate` can be used in place of the corresponding +`mamba` commands. Use the same environment to build and run both packages. +A C++ compiler is needed when building `python-ace` from source; its Python +build dependencies, including CMake and Ninja, are installed automatically. -* Activate the environment with either: `source activate ace` or `mamba activate ace`. To deactivate the environment, use the command `mamba deactivate` +## Install the fitting backend -### Using `conda` -> **Note:** Due to recent changes in Anaconda's licensing, please ensure you are permitted to use it. The `mamba` package manager might be a preferable alternative. +Install the current backend from source: -To create the environment with `conda`: ```bash -conda create -n ace python=3.9 +git clone https://github.com/ICAMS/grace-tensorpotential.git +cd grace-tensorpotential +python -m pip install . +cd .. ``` -## Installation of `tensorpotential` +This backend must include the PACE metrics scalar-conversion fix for NumPy +2.4 and newer. The released TensorPotential 0.6.2.1 also caps TensorFlow at +2.20; TensorFlow 2.21 needs the updated backend dependency bound. Updating +`python-ace` alone does not update either of these backend requirements. -`tensorpotential` allows for the GPU accelerated optimization of the ACE potential using [TensorFlow](https://www.tensorflow.org/). -However, it is recommended to use it even if you don't have a GPU available. +The updated backend permits TensorFlow releases below 2.22. To select the +2.21 series explicitly, install matching TensorFlow and legacy Keras versions: +```bash +python -m pip install 'tensorflow~=2.21.0' 'tf-keras~=2.21.0' +``` -Install it using the following commands: +On Linux the backend dependencies include `tensorflow[and-cuda]`. GPU execution +also requires a compatible NVIDIA driver. CPU fitting is available with +`CUDA_VISIBLE_DEVICES=-1`. TensorFlow 2.21 publishes wheels for Python +3.10–3.13; Python 3.14 is not covered by this installation recipe. -* Install Tensorflow (newer version should be also compatible) -``` -pip install tensorflow==2.8.0 -``` -or to have CUDA support in latest versions of TensorFlow -``` -pip install tensorflow[and-cuda] -``` +## Install pacemaker -* Download the `tensorpotential` from [this repository](https://github.com/ICAMS/TensorPotential). -Clone with -``` -git clone https://github.com/ICAMS/TensorPotential.git -cd TensorPotential -``` -or download -``` -wget https://github.com/ICAMS/TensorPotential/archive/refs/heads/main.zip -unzip main.zip -cd TensorPotential-main -``` +For the modern source branch: -* Run installation script -``` -pip install --upgrade . +```bash +git clone --branch feature/build-speedup https://github.com/ICAMS/python-ace.git +cd python-ace +python -m pip install . ``` -or (for more installation details) + +When working in an existing checkout, run `python -m pip install .` there. +This build includes the maxvol extension used by the active-learning tools; +a separate `maxvolpy` installation is unnecessary. + +Check the installed versions and command-line tools: + +```bash +python -m pip check +python -c 'import pyace, tensorpotential, tensorflow as tf; print(pyace.__version__, tensorpotential.__version__, tf.__version__)' +pacemaker --help +pace_info --help ``` -python setup.py install + +To run the test suite from the `python-ace` checkout: + +```bash +python -m pip install pytest +CUDA_VISIBLE_DEVICES=-1 python -m pytest tests/ --runtensorpot ``` -## Installation of `pacemaker` and `pyace` +The test workflow is configured for Python 3.10–3.13 with TensorFlow 2.20 and +2.21 on Linux CPUs, plus Python 3.9 with TensorFlow 2.20. GPU execution needs +separate validation on the target hardware. -The `pyace` (aka `python-ace`) package is located in [this repository](https://github.com/ICAMS/python-ace). -It contains the `pacemaker` tools and other Python wrappers and utilities. +### Tested development environment -To install `pyace`: +The source refresh was checked on 29 October 2026 with Python 3.12.15, +TensorFlow 2.21.0, tf-keras 2.21.0, NumPy 2.5.4, and pandas 2.3.3: -* Download `pyace` from [this repository](https://github.com/ICAMS/python-ace). Clone with -``` -git clone https://github.com/ICAMS/python-ace.git -cd python-ace -``` -or download -``` -wget https://github.com/ICAMS/python-ace/archive/refs/heads/master.zip -cd python-ace-master -``` +| Execution target | Validation | +| --- | --- | +| Linux CPU | 294 tests and both Cu–I and Ethanol CLI workflows passed | +| NVIDIA H200 NVL, driver 595.58.03 | All 9 fitting tests passed; 185 fitting calls verified on GPU | +| NVIDIA A100 80 GB PCIe, driver 575.57.08 | All 9 fitting tests passed; 185 fitting calls verified on GPU | -* Run installation script -``` -pip install --upgrade . -``` -or (for more installation details) -``` -python setup.py install -``` +The GPU checks verify device placement of energies, forces, and coefficient +gradients as well as the fitting-test results. They cover short fits, rather +than long-running memory stability or performance. These results require the +source fixes described above and do not describe an unmodified TensorPotential +0.6.2.1 installation. -Now, `pacemaker` and other tools (`pace_yaml2yace`, `pace_info`, `pace_activeset`) should be available from the terminal, if corresponding conda environment is loaded. +## Troubleshooting -## Known installation issues +### GPU validation -### Segmentation fault -If you see `Segmentation fault` error message, then check that you are using correct version of Python from corresponding conda environment, -i.e. check that `which python` points to right location inside conda environment. +To check fitting on a selected GPU from the source checkout: -### TypeError: Descriptors cannot not be created directly -If you see this error message -``` -TypeError: Descriptors cannot not be created directly. -If this call came from a _pb2.py file, your generated code is out of date and must be regenerated with protoc >= 3.19.0. -If you cannot immediately regenerate your protos, some other possible workarounds are: - 1. Downgrade the protobuf package to 3.20.x or lower. - 2. Set PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python (but this will use pure-Python parsing and will be much slower).``` -``` -then try to downgrade `protobuf` package, i.e. +```bash +CUDA_VISIBLE_DEVICES=0 TF_FORCE_GPU_ALLOW_GROWTH=true python tools/validate_gpu.py ``` -pip install protobuf==3.20.* + +This runs the TensorFlow fitting tests and checks that energies, forces, and +coefficient gradients reside on the GPU. It fails if no GPU is detected or if +those results fall back to the CPU. Select one GPU; pacemaker does not fit on +multiple GPUs in parallel. Memory growth keeps the test from reserving all +available device memory. + +If TensorFlow reports `Cannot dlopen some GPU libraries`, an existing system +CUDA installation may take precedence over the libraries installed by pip. +For the current shell, put the environment's CUDA library directories first: + +```bash +pace_cuda_libraries=$(python - <<'PY' +from pathlib import Path +import sysconfig +print(':'.join(str(path) for path in sorted( + (Path(sysconfig.get_paths()['purelib']) / 'nvidia').glob('*/lib') +))) +PY +) +export LD_LIBRARY_PATH="$pace_cuda_libraries${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" ``` + +Then rerun the GPU check. This changes the shell environment and leaves the +system CUDA installation untouched. + +### Python and dependency errors + +If fitting raises `TypeError: only 0-dimensional arrays can be converted to +Python scalars`, check that both `python-ace` and the backend include the +NumPy scalar-conversion fix, then rebuild the installed packages. + +For an import error or segmentation fault, check `which python` and rebuild +the extensions using the interpreter from the active environment. Avoid mixing +extensions built for different Python versions. + +For a protobuf descriptor error, check `python -m pip check` and reinstall a +consistent TensorFlow environment. The old TensorFlow 2.5/2.8 and protobuf +3.20 instructions do not describe the current dependency set. diff --git a/docs/pacemaker/quickstart.md b/docs/pacemaker/quickstart.md index 4e449d0..cda7bed 100644 --- a/docs/pacemaker/quickstart.md +++ b/docs/pacemaker/quickstart.md @@ -5,7 +5,7 @@ contains structural information as well as corresponding energies and forces tha Input file contains details about desired ACE potential configuration and various parameters influencing optimization process. In this section we will describe the format of the fitting dataset, we will run a fit with an example dataset and -overview the output produced by `pacemaker`. Input parameters are detailed in the [section](inputfile.md#Input_file) below. +overview the output produced by `pacemaker`. Input parameters are detailed in the [section](inputfile.md#input_file) below. ## Automatic DFT data collection @@ -23,7 +23,7 @@ Both commands will scan through all folders and subfolders and collect DFT free and make a single atom corrections. Resulting dataset will be stored into `collected.pckl.gzip` file. If you need more flexibility for DFT dataset manipulation, -please check [Manual fitting dataset preparation](#manual_fitting_dataset_preparation). +please check [Manual fitting dataset preparation](#optional_manual_fitting_dataset_preparation). ## Automatic input file generation @@ -165,7 +165,7 @@ Note, that `tensorpotential` can be used without a GPU as well. During and after the fitting `pacemaker` produces several outputs, including: -- `interim_potential_X.yaml`: current state of the potential at each iteration of [fit cycle](inputfile.md#fitting-settings) (i.g. X=0, 1, ...) +- `interim_potential_X.yaml`: current state of the potential at each iteration of [fit cycle](inputfile.md#fitting_settings) (i.g. X=0, 1, ...) - `interim_potential_best_cycle.yaml`: best out of X interim potentials - `log.txt`: log file containing all current information including summary of the optimization steps. - `report`: folder containing figures displaying various error statistics and distributions. @@ -178,7 +178,7 @@ There are two main types of the information in the log file: ```text Iteration #999 (1052 evals): Loss: 0.000192 | RMSE Energy(low): 17.95 (16.79) meV/at | Forces(low): 7.89 (7.04) meV/A | Time/eval: 517.83 mcs/at ``` -where `Iteration` is the index of the optimization step performed by the [optimizer](inputfile.md#fitting-settings) +where `Iteration` is the index of the optimization step performed by the [optimizer](inputfile.md#fitting_settings) (number in parentheses shows the number of function evaluation calls done by optimizaer), `Loss` is the current value of the loss function, `RMSE Energy/Forces` is the current root mean-squared error for energy/forces wrt. training dataset (numbers in paretheses show corresponding values for the structures which @@ -206,8 +206,8 @@ Number of params./funcs: 232/86 Avg. time: ``` -Every [display_step](inputfile.md#backend-specification) the summary of fit statistics is printed out. It displays the total -loss function value and contributions to it from energy, forces and other [regularizations parameters](inputfile.md#fitting-settings). +Every [display_step](inputfile.md#backend_specification) the summary of fit statistics is printed out. It displays the total +loss function value and contributions to it from energy, forces and other [regularizations parameters](inputfile.md#fitting_settings). In addition to RMSE, mean-absolute error (MAE) and maximum absolute error (MAX_AE) are also printed. @@ -219,7 +219,7 @@ Fitted potential can be used for calculations both in [LAMMPS](https://docs.lamm ### LAMMPS -Using potential with [LAMMPS](https://www.lammps.org/) requires its [conversion](utilities.md#potential-conversion) into **YACE** format with command +Using potential with [LAMMPS](https://www.lammps.org/) requires its [conversion](utilities.md#potential_conversion) into **YACE** format with command ```asm pace_yaml2yace output_potential.yaml ``` diff --git a/docs/requirements.txt b/docs/requirements.txt index 540e569..cdc69a4 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,3 @@ -mkdocs -mkdocs-material -mkdocs-jupyter +mkdocs>=1.6,<2 +mkdocs-material>=9.6,<10 +mkdocs-jupyter>=0.25,<1 diff --git a/mkdocs.yml b/mkdocs.yml index 43bb87e..e3672f0 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,4 +1,11 @@ site_name: pacemaker +site_url: https://python-ace.readthedocs.io/en/latest/ +repo_url: https://github.com/ICAMS/python-ace + +validation: + links: + unrecognized_links: warn + anchors: warn theme: name: "material" @@ -18,6 +25,7 @@ extra_javascript: - https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js nav: + - Home: index.md - Installation: pacemaker/install.md - Quick start: pacemaker/quickstart.md - Pacemaker workflow: pacemaker/workflow.md diff --git a/src/pyace/metrics_aggregator.py b/src/pyace/metrics_aggregator.py index 2289779..1cfcf4f 100644 --- a/src/pyace/metrics_aggregator.py +++ b/src/pyace/metrics_aggregator.py @@ -27,7 +27,8 @@ def to_FitMetricsDict(self): :return: fit metrics dictionary """ - regularization_loss = [float(r_comp * r_weight) for r_comp, r_weight in zip(self.regs, self.reg_weights)] + regularization_loss = [float(np.asarray(r_comp * r_weight).item()) + for r_comp, r_weight in zip(self.regs, self.reg_weights)] l1 = regularization_loss[0] l2 = regularization_loss[1] smoothness_reg_loss = regularization_loss[2:] diff --git a/tests/test_metrics_aggregator.py b/tests/test_metrics_aggregator.py new file mode 100644 index 0000000..73e7896 --- /dev/null +++ b/tests/test_metrics_aggregator.py @@ -0,0 +1,31 @@ +import numpy as np +import pandas as pd +import pytest + +from pyace.metrics_aggregator import FitMetrics + + +@pytest.mark.parametrize("shape", [(4,), (4, 1)]) +def test_regularization_contributions_are_scalars(shape): + metrics = FitMetrics( + w_e=np.ones((1, 1)), w_f=np.ones((1, 1)), + e_scale=0.5, f_scale=0.5, ncoefs=2, + regs=np.array([2.0, 3.0, 4.0, 5.0]).reshape(shape), + ) + metrics.loss = 0.0 + metrics.eval_time = 0.0 + metrics.reg_weights = np.array([0.1, 0.2, 0.3, 0.4]).reshape(shape) + metrics.compute_metrics( + de=np.zeros((1, 1)), de_pa=np.zeros((1, 1)), + df=np.zeros((1, 3)), nat=np.ones((1, 1), dtype=int), + dataframe=pd.DataFrame({"energy_corrected": [0.0]}), + ) + + result = metrics.to_FitMetricsDict() + + assert result["l1_reg_contrib"] == pytest.approx(0.2) + assert result["l2_reg_contrib"] == pytest.approx(0.6) + assert result["extra_regularization_contrib"] == pytest.approx([1.2, 2.0]) + assert type(result["l1_reg_contrib"]) is float + assert type(result["l2_reg_contrib"]) is float + assert all(type(value) is float for value in result["extra_regularization_contrib"]) diff --git a/tools/validate_gpu.py b/tools/validate_gpu.py new file mode 100644 index 0000000..c0f252b --- /dev/null +++ b/tools/validate_gpu.py @@ -0,0 +1,61 @@ +"""Run PACE fitting tests and verify native energy/force/gradient placement.""" +import json +import os +from pathlib import Path +import sys + +os.environ.setdefault("TF_USE_LEGACY_KERAS", "1") +os.environ.setdefault("TF_FORCE_GPU_ALLOW_GROWTH", "true") +os.environ.setdefault("OMP_NUM_THREADS", "2") +os.environ.setdefault("TF_NUM_INTRAOP_THREADS", "2") +os.environ.setdefault("TF_NUM_INTEROP_THREADS", "2") + +import tensorpotential +import tensorflow as tf +import pytest +from tensorpotential.compat.pace.tensorpot import TensorPotential + +physical = tf.config.list_physical_devices("GPU") +assert len(physical) == 1, f"Expected exactly one selected GPU, found {physical}" +for device in physical: + tf.config.experimental.set_memory_growth(device, True) +tf.config.set_soft_device_placement(False) +with tf.device("/GPU:0"): + product = tf.linalg.matmul(tf.ones((128, 128), tf.float64), tf.ones((128, 128), tf.float64)) +assert "GPU:0" in product.device, product.device +assert float(product[0, 0]) == 128.0 +print("GPU_PREFLIGHT", json.dumps({ + "tensorflow": tf.__version__, "tensorpotential": tensorpotential.__version__, + "details": tf.config.experimental.get_device_details(physical[0]), + "matmul_device": product.device, +}), flush=True) + +originals = {name: getattr(TensorPotential, name) for name in ("native_fit", "external_fit")} +placements = [] + +def check_result(result): + devices = {name: value.device for name, value in zip(("loss", "gradient", "energy", "forces"), result)} + for name in ("gradient", "energy", "forces"): + assert "GPU:0" in devices[name], f"PACE {name} fell back to {devices[name]}" + if not placements: + print("PACE_NATIVE_PLACEMENT", json.dumps(devices), flush=True) + placements.append(devices) + return result + +def checked_fit(name): + def call(self, *args, **kwargs): + return check_result(originals[name](self, *args, **kwargs)) + return call + +for name in originals: + setattr(TensorPotential, name, checked_fit(name)) +args = sys.argv[1:] or ["tests/", "--runtensorpot", "-m", "tensorpot", "-q"] +status = pytest.main(args) +summary = { + "pytest_exit_code": int(status), "fitting_calls_on_gpu": len(placements), + "first_placement": placements[0] if placements else None, + "peak_gpu_memory": tf.config.experimental.get_memory_info("GPU:0"), +} +print("GPU_VALIDATION_RESULT", json.dumps(summary), flush=True) +assert placements, "No PACE fitting calls were exercised" +sys.exit(status) From f12a1b1f2e43e8809a5e808738bc8f8339e4727c Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Thu, 29 Oct 2026 23:36:23 +0100 Subject: [PATCH 29/30] Pin compatibility CI to merged TensorPotential fix --- .github/workflows/test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 41586cf..34dfaed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,6 +24,9 @@ jobs: CUDA_VISIBLE_DEVICES: "-1" TF_USE_LEGACY_KERAS: "1" CMAKE_BUILD_PARALLEL_LEVEL: "2" + OMP_NUM_THREADS: "2" + TF_NUM_INTRAOP_THREADS: "2" + TF_NUM_INTEROP_THREADS: "2" steps: - uses: actions/checkout@v4 @@ -39,6 +42,8 @@ jobs: uses: actions/checkout@v4 with: repository: ICAMS/grace-tensorpotential + # Includes the NumPy scalar fix and TensorFlow 2.21 bound from PR #43. + ref: 9ccb9090f53292a836cbbc3de4cadcae54c7ce1a path: tensorpotential - name: Install tensorpotential run: | @@ -50,6 +55,7 @@ jobs: - name: Test with python-ace with tensorpotential run: | python -m pip check + python -c 'import pyace, tensorpotential, tensorflow as tf, numpy as np; print("Versions:", pyace.__version__, tensorpotential.__version__, tf.__version__, np.__version__)' python -m pytest tests/ --runtensorpot - name: Integration test of CLI run: | From f0f2a4e89184861f2dc58f857255c12daa7408c8 Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Sat, 31 Oct 2026 15:16:37 +0100 Subject: [PATCH 30/30] Document released GRACE backend installation from PyPI --- README.md | 6 ++--- docs/pacemaker/install.md | 53 +++++++++++++++++++++++---------------- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index eb70a41..288951f 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,16 @@ # Description This repository contains `pacemaker` - a tool for fitting of interatomic potentials in a general nonlinear Atomic Cluster Expansion (ACE) form. -It is based on the `pyace` (this repo) and `tensorpotential`([another repo](https://github.com/ICAMS/TensorPotential)) Python packages. +It is based on the `pyace` (this repo) and `tensorpotential` ([GRACE TensorPotential](https://github.com/ICAMS/grace-tensorpotential)) Python packages. # Installation -Please follow [these](https://pacemaker.readthedocs.io/en/latest/pacemaker/install/) instructions. +Please follow the [installation instructions](https://python-ace.readthedocs.io/en/latest/pacemaker/install/). The GRACE backend is installed from PyPI as `tensorpotential`; the pacemaker distribution is `python-ace`. # Documentation -Please see the `docs` folder for documentation or check [online documentation](https://pacemaker.readthedocs.io/) +Please see the `docs` folder for documentation or check [online documentation](https://python-ace.readthedocs.io/) # Citation diff --git a/docs/pacemaker/install.md b/docs/pacemaker/install.md index 108d784..d4d31a7 100644 --- a/docs/pacemaker/install.md +++ b/docs/pacemaker/install.md @@ -23,26 +23,18 @@ build dependencies, including CMake and Ninja, are installed automatically. ## Install the fitting backend -Install the current backend from source: +Install the GRACE backend from PyPI. Its distribution name is +`tensorpotential`; `grace-tensorpotential` is the GitHub repository name. +The following released-package combination works around the PACE metrics +scalar-conversion bug in TensorPotential 0.6.2.1: ```bash -git clone https://github.com/ICAMS/grace-tensorpotential.git -cd grace-tensorpotential -python -m pip install . -cd .. +python -m pip install 'tensorpotential==0.6.2.1' 'numpy>=2,<2.4' 'tensorflow~=2.20.0' 'tf-keras~=2.20.0' ``` -This backend must include the PACE metrics scalar-conversion fix for NumPy -2.4 and newer. The released TensorPotential 0.6.2.1 also caps TensorFlow at -2.20; TensorFlow 2.21 needs the updated backend dependency bound. Updating -`python-ace` alone does not update either of these backend requirements. - -The updated backend permits TensorFlow releases below 2.22. To select the -2.21 series explicitly, install matching TensorFlow and legacy Keras versions: - -```bash -python -m pip install 'tensorflow~=2.21.0' 'tf-keras~=2.21.0' -``` +Keep NumPy below 2.4 when using that backend release. The NumPy fix and +TensorFlow 2.21 support have been merged upstream but are not included in +PyPI's 0.6.2.1 release. See the development installation below to use them. On Linux the backend dependencies include `tensorflow[and-cuda]`. GPU execution also requires a compatible NVIDIA driver. CPU fitting is available with @@ -51,18 +43,35 @@ also requires a compatible NVIDIA driver. CPU fitting is available with ## Install pacemaker -For the modern source branch: +Install the published pacemaker release candidate explicitly: + +```bash +python -m pip install 'python-ace==0.4.0rc1' +``` + +The version is explicit because an unqualified `pip install python-ace` +selects the older stable 0.2.8 release. The modern package includes the maxvol +extension used by the active-learning tools; a separate `maxvolpy` +installation is unnecessary. + +### Development installation + +To use the merged TensorFlow 2.21 and NumPy fixes, install the backend from +GitHub and pacemaker from the default `master` branch: + +```bash +python -m pip install --upgrade 'tensorpotential @ git+https://github.com/ICAMS/grace-tensorpotential.git' 'tensorflow~=2.21.0' 'tf-keras~=2.21.0' +python -m pip install --upgrade 'python-ace @ git+https://github.com/ICAMS/python-ace.git' +``` + +For a source checkout or to run the tests: ```bash -git clone --branch feature/build-speedup https://github.com/ICAMS/python-ace.git +git clone https://github.com/ICAMS/python-ace.git cd python-ace python -m pip install . ``` -When working in an existing checkout, run `python -m pip install .` there. -This build includes the maxvol extension used by the active-learning tools; -a separate `maxvolpy` installation is unnecessary. - Check the installed versions and command-line tools: ```bash